Change List interface and add ListR optional interface

This simplifies the implementation of remotes.  The only required
interface is now `List` which is a simple one level directory list.

Optionally remotes may implement `ListR` if they have an efficient way
of doing a recursive list.

googlecloudstorage/googlecloudstorage.go

@@ -308,27 +308,28 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 //
 // dir is the starting directory, "" for root
 //
-// If directories is set it only sends directories
-func (f *Fs) list(dir string, level int, fn listFn) error {
+// Set recurse to read sub directories
+func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 	root := f.root
 	rootLength := len(root)
 	if dir != "" {
 		root += dir + "/"
 	}
 	list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks)
-	switch level {
-	case 1:
+	if !recurse {
 		list = list.Delimiter("/")
-	case fs.MaxLevel:
-	default:
-		return fs.ErrorLevelNotSupported
 	}
 	for {
 		objects, err := list.Do()
 		if err != nil {
+			if gErr, ok := err.(*googleapi.Error); ok {
+				if gErr.Code == http.StatusNotFound {
+					err = fs.ErrorDirNotFound
+				}
+			}
 			return err
 		}
-		if level == 1 {
+		if !recurse {
 			var object storage.Object
 			for _, prefix := range objects.Prefixes {
 				if !strings.HasSuffix(prefix, "/") {
@@ -359,94 +360,120 @@ func (f *Fs) list(dir string, level int, fn listFn) error {
 	return nil
 }
 
-// listFiles lists files and directories to out
-func (f *Fs) listFiles(out fs.ListOpts, dir string) {
-	defer out.Finished()
-	if f.bucket == "" {
-		out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
-		return
+// Convert a list item into a BasicInfo
+func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory bool) (fs.BasicInfo, error) {
+	if isDirectory {
+		d := &fs.Dir{
+			Name:  remote,
+			Bytes: int64(object.Size),
+			Count: 0,
+		}
+		return d, nil
 	}
+	o, err := f.newObjectWithInfo(remote, object)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// listDir lists a single directory
+func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	// List the objects
-	err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
-		if isDirectory {
-			dir := &fs.Dir{
-				Name:  remote,
-				Bytes: int64(object.Size),
-				Count: 0,
-			}
-			if out.AddDir(dir) {
-				return fs.ErrorListAborted
-			}
-		} else {
-			o, err := f.newObjectWithInfo(remote, object)
-			if err != nil {
-				return err
-			}
-			if out.Add(o) {
-				return fs.ErrorListAborted
-			}
+	err = f.list(dir, false, func(remote string, object *storage.Object, isDirectory bool) error {
+		entry, err := f.itemToDirEntry(remote, object, isDirectory)
+		if err != nil {
+			return err
+		}
+		if entry != nil {
+			entries = append(entries, entry)
 		}
 		return nil
 	})
 	if err != nil {
-		if gErr, ok := err.(*googleapi.Error); ok {
-			if gErr.Code == http.StatusNotFound {
-				err = fs.ErrorDirNotFound
-			}
-		}
-		out.SetError(err)
+		return nil, err
 	}
+	return entries, err
 }
 
-// listBuckets lists the buckets to out
-func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
-	defer out.Finished()
+// listBuckets lists the buckets
+func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	if dir != "" {
-		out.SetError(fs.ErrorListOnlyRoot)
-		return
+		return nil, fs.ErrorListBucketRequired
 	}
 	if f.projectNumber == "" {
-		out.SetError(errors.New("can't list buckets without project number"))
-		return
+		return nil, errors.New("can't list buckets without project number")
 	}
 	listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
 	for {
 		buckets, err := listBuckets.Do()
 		if err != nil {
-			out.SetError(err)
-			return
+			return nil, err
 		}
 		for _, bucket := range buckets.Items {
-			dir := &fs.Dir{
+			d := &fs.Dir{
 				Name:  bucket.Name,
 				Bytes: 0,
 				Count: 0,
 			}
-			if out.AddDir(dir) {
-				return
-			}
+			entries = append(entries, d)
 		}
 		if buckets.NextPageToken == "" {
 			break
 		}
 		listBuckets.PageToken(buckets.NextPageToken)
 	}
+	return entries, nil
 }
 
-// List lists the path to out
-func (f *Fs) List(out fs.ListOpts, dir string) {
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if f.bucket == "" {
-		f.listBuckets(out, dir)
-	} else {
-		f.listFiles(out, dir)
+		return f.listBuckets(dir)
 	}
-	return
+	return f.listDir(dir)
 }
 
 // ListR lists the objects and directories of the Fs starting
 // from dir recursively into out.
-func (f *Fs) ListR(out fs.ListOpts, dir string) {
-	f.List(out, dir) // FIXME
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+//
+// Don't implement this unless you have a more efficient way
+// of listing recursively that doing a directory traversal.
+func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
+	if f.bucket == "" {
+		return fs.ErrorListBucketRequired
+	}
+	list := fs.NewListRHelper(callback)
+	err = f.list(dir, true, func(remote string, object *storage.Object, isDirectory bool) error {
+		entry, err := f.itemToDirEntry(remote, object, isDirectory)
+		if err != nil {
+			return err
+		}
+		return list.Add(entry)
+	})
+	if err != nil {
+		return err
+	}
+	return list.Flush()
 }
 
 // Put the object into the bucket

googlecloudstorage/googlecloudstorage_test.go

@@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
 func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
 func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
 func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
+func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
 func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
 func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
 func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
 func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
 func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
 func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
+func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
 func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
+func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
 func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
+func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
 func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
+func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
 func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
 func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
 func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }