dropbox: add ListP interface - #4788

backend/dropbox/dropbox.go

@@ -47,6 +47,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
@@ -834,7 +835,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // listSharedFolders lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
-func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
+func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) error) (err error) {
 	started := false
 	var res *sharing.ListFoldersResult
 	for {
@@ -847,7 +848,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
-				return nil, err
+				return err
 			}
 			started = true
 		} else {
@@ -859,15 +860,15 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
-				return nil, fmt.Errorf("list continue: %w", err)
+				return fmt.Errorf("list continue: %w", err)
 			}
 		}
 		for _, entry := range res.Entries {
 			leaf := f.opt.Enc.ToStandardName(entry.Name)
 			d := fs.NewDir(leaf, time.Time{}).SetID(entry.SharedFolderId)
-			entries = append(entries, d)
+			err = callback(d)
 			if err != nil {
-				return nil, err
+				return err
 			}
 		}
 		if res.Cursor == "" {
@@ -875,21 +876,25 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
 		}
 	}
 
-	return entries, nil
+	return nil
 }
 
 // findSharedFolder find the id for a given shared folder name
 // somewhat annoyingly there is no endpoint to query a shared folder by it's name
 // so our only option is to iterate over all shared folders
 func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
-	entries, err := f.listSharedFolders(ctx)
-	if err != nil {
-		return "", err
-	}
-	for _, entry := range entries {
+	errFoundFile := errors.New("found file")
+	err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
 		if entry.(*fs.Dir).Remote() == name {
-			return entry.(*fs.Dir).ID(), nil
+			id = entry.(*fs.Dir).ID()
+			return errFoundFile
 		}
+		return nil
+	})
+	if errors.Is(err, errFoundFile) {
+		return id, nil
+	} else if err != nil {
+		return "", err
 	}
 	return "", fs.ErrorDirNotFound
 }
@@ -908,7 +913,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
 
 // listReceivedFiles lists shared the user as access to (note this means individual
 // files not files contained in shared folders)
-func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
+func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) error) (err error) {
 	started := false
 	var res *sharing.ListFilesResult
 	for {
@@ -921,7 +926,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
-				return nil, err
+				return err
 			}
 			started = true
 		} else {
@@ -933,7 +938,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
-				return nil, fmt.Errorf("list continue: %w", err)
+				return fmt.Errorf("list continue: %w", err)
 			}
 		}
 		for _, entry := range res.Entries {
@@ -946,26 +951,33 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
 				modTime: *entry.TimeInvited,
 			}
 			if err != nil {
-				return nil, err
+				return err
+			}
+			err = callback(o)
+			if err != nil {
+				return err
 			}
-			entries = append(entries, o)
 		}
 		if res.Cursor == "" {
 			break
 		}
 	}
-	return entries, nil
+	return nil
 }
 
 func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
-	files, err := f.listReceivedFiles(ctx)
-	if err != nil {
-		return nil, err
-	}
-	for _, entry := range files {
+	errFoundFile := errors.New("found file")
+	err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
 		if entry.(*Object).remote == name {
-			return entry.(*Object), nil
+			o = entry.(*Object)
+			return errFoundFile
 		}
+		return nil
+	})
+	if errors.Is(err, errFoundFile) {
+		return o, nil
+	} else if err != nil {
+		return nil, err
 	}
 	return nil, fs.ErrorObjectNotFound
 }
@@ -980,11 +992,37 @@ func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err er
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	list := list.NewHelper(callback)
 	if f.opt.SharedFiles {
-		return f.listReceivedFiles(ctx)
+		err := f.listReceivedFiles(ctx, list.Add)
+		if err != nil {
+			return err
+		}
+		return list.Flush()
 	}
 	if f.opt.SharedFolders {
-		return f.listSharedFolders(ctx)
+		err := f.listSharedFolders(ctx, list.Add)
+		if err != nil {
+			return err
+		}
+		return list.Flush()
 	}
 
 	root := f.slashRoot
@@ -1014,7 +1052,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 						err = fs.ErrorDirNotFound
 					}
 				}
-				return nil, err
+				return err
 			}
 			started = true
 		} else {
@@ -1026,7 +1064,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
-				return nil, fmt.Errorf("list continue: %w", err)
+				return fmt.Errorf("list continue: %w", err)
 			}
 		}
 		for _, entry := range res.Entries {
@@ -1051,14 +1089,20 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
 				d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
-				entries = append(entries, d)
+				err = list.Add(d)
+				if err != nil {
+					return err
+				}
 			} else if fileInfo != nil {
 				o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
 				if err != nil {
-					return nil, err
+					return err
 				}
 				if o.(*Object).exportType.listable() {
-					entries = append(entries, o)
+					err = list.Add(o)
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
@@ -1066,7 +1110,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			break
 		}
 	}
-	return entries, nil
+	return list.Flush()
 }
 
 // Put the object
@@ -2087,6 +2131,7 @@ var (
 	_ fs.Mover        = (*Fs)(nil)
 	_ fs.PublicLinker = (*Fs)(nil)
 	_ fs.DirMover     = (*Fs)(nil)
+	_ fs.ListPer      = (*Fs)(nil)
 	_ fs.Abouter      = (*Fs)(nil)
 	_ fs.Shutdowner   = &Fs{}
 	_ fs.Object       = (*Object)(nil)