Change List interface and add ListR optional interface

This simplifies the implementation of remotes.  The only required
interface is now `List` which is a simple one level directory list.

Optionally remotes may implement `ListR` if they have an efficient way
of doing a recursive list.

s3/s3.go

@@ -470,20 +470,16 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error
 //
 // dir is the starting directory, "" for root
 //
-// Level is the level of the recursion
-func (f *Fs) list(dir string, level int, fn listFn) error {
+// Set recurse to read sub directories
+func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 	root := f.root
 	if dir != "" {
 		root += dir + "/"
 	}
 	maxKeys := int64(listChunkSize)
 	delimiter := ""
-	switch level {
-	case 1:
+	if !recurse {
 		delimiter = "/"
-	case fs.MaxLevel:
-	default:
-		return fs.ErrorLevelNotSupported
 	}
 	var marker *string
 	for {
@@ -497,10 +493,15 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
 		}
 		resp, err := f.c.ListObjects(&req)
 		if err != nil {
+			if awsErr, ok := err.(awserr.RequestFailure); ok {
+				if awsErr.StatusCode() == http.StatusNotFound {
+					err = fs.ErrorDirNotFound
+				}
+			}
 			return err
 		}
 		rootLength := len(f.root)
-		if level == 1 {
+		if !recurse {
 			for _, commonPrefix := range resp.CommonPrefixes {
 				if commonPrefix.Prefix == nil {
 					fs.Logf(f, "Nil common prefix received")
@@ -546,90 +547,116 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
 	return nil
 }
 
-// listFiles lists files and directories to out
-func (f *Fs) listFiles(out fs.ListOpts, dir string) {
-	defer out.Finished()
-	if f.bucket == "" {
-		// Return no objects at top level list
-		out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
-		return
+// Convert a list item into a BasicInfo
+func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool) (fs.BasicInfo, error) {
+	if isDirectory {
+		size := int64(0)
+		if object.Size != nil {
+			size = *object.Size
+		}
+		d := &fs.Dir{
+			Name:  remote,
+			Bytes: size,
+			Count: 0,
+		}
+		return d, nil
 	}
+	o, err := f.newObjectWithInfo(remote, object)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// listDir lists files and directories to out
+func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	// List the objects and directories
-	err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
-		if isDirectory {
-			size := int64(0)
-			if object.Size != nil {
-				size = *object.Size
-			}
-			dir := &fs.Dir{
-				Name:  remote,
-				Bytes: size,
-				Count: 0,
-			}
-			if out.AddDir(dir) {
-				return fs.ErrorListAborted
-			}
-		} else {
-			o, err := f.newObjectWithInfo(remote, object)
-			if err != nil {
-				return err
-			}
-			if out.Add(o) {
-				return fs.ErrorListAborted
-			}
+	err = f.list(dir, false, func(remote string, object *s3.Object, isDirectory bool) error {
+		entry, err := f.itemToDirEntry(remote, object, isDirectory)
+		if err != nil {
+			return err
+		}
+		if entry != nil {
+			entries = append(entries, entry)
 		}
 		return nil
 	})
 	if err != nil {
-		if awsErr, ok := err.(awserr.RequestFailure); ok {
-			if awsErr.StatusCode() == http.StatusNotFound {
-				err = fs.ErrorDirNotFound
-			}
-		}
-		out.SetError(err)
+		return nil, err
 	}
+	return entries, nil
 }
 
 // listBuckets lists the buckets to out
-func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
-	defer out.Finished()
+func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	if dir != "" {
-		out.SetError(fs.ErrorListOnlyRoot)
-		return
+		return nil, fs.ErrorListBucketRequired
 	}
 	req := s3.ListBucketsInput{}
 	resp, err := f.c.ListBuckets(&req)
 	if err != nil {
-		out.SetError(err)
-		return
+		return nil, err
 	}
 	for _, bucket := range resp.Buckets {
-		dir := &fs.Dir{
+		d := &fs.Dir{
 			Name:  aws.StringValue(bucket.Name),
 			When:  aws.TimeValue(bucket.CreationDate),
 			Bytes: -1,
 			Count: -1,
 		}
-		if out.AddDir(dir) {
-			break
-		}
+		entries = append(entries, d)
 	}
+	return entries, nil
 }
 
-// List lists files and directories to out
-func (f *Fs) List(out fs.ListOpts, dir string) {
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if f.bucket == "" {
-		f.listBuckets(out, dir)
-	} else {
-		f.listFiles(out, dir)
+		return f.listBuckets(dir)
 	}
-	return
+	return f.listDir(dir)
 }
 
 // ListR lists the objects and directories of the Fs starting
 // from dir recursively into out.
-func (f *Fs) ListR(out fs.ListOpts, dir string) {
-	f.List(out, dir) // FIXME
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+//
+// Don't implement this unless you have a more efficient way
+// of listing recursively that doing a directory traversal.
+func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
+	if f.bucket == "" {
+		return fs.ErrorListBucketRequired
+	}
+	list := fs.NewListRHelper(callback)
+	err = f.list(dir, true, func(remote string, object *s3.Object, isDirectory bool) error {
+		entry, err := f.itemToDirEntry(remote, object, isDirectory)
+		if err != nil {
+			return err
+		}
+		return list.Add(entry)
+	})
+	if err != nil {
+		return err
+	}
+	return list.Flush()
 }
 
 // Put the Object into the bucket