diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index f67a9e261..25cfe2e9d 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -47,6 +47,7 @@ import ( "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/batcher" "github.com/rclone/rclone/lib/encoder" @@ -834,7 +835,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // listSharedFolders lists all available shared folders mounted and not mounted // we'll need the id later so we have to return them in original format -func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) { +func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) error) (err error) { started := false var res *sharing.ListFoldersResult for { @@ -847,7 +848,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err return shouldRetry(ctx, err) }) if err != nil { - return nil, err + return err } started = true } else { @@ -859,15 +860,15 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err return shouldRetry(ctx, err) }) if err != nil { - return nil, fmt.Errorf("list continue: %w", err) + return fmt.Errorf("list continue: %w", err) } } for _, entry := range res.Entries { leaf := f.opt.Enc.ToStandardName(entry.Name) d := fs.NewDir(leaf, time.Time{}).SetID(entry.SharedFolderId) - entries = append(entries, d) + err = callback(d) if err != nil { - return nil, err + return err } } if res.Cursor == "" { @@ -875,21 +876,25 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err } } - return entries, nil + return nil } // findSharedFolder find the id for a given shared folder name // somewhat annoyingly there is no endpoint to query a shared folder by it's name // so our only option is to iterate over all shared folders func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) { - entries, err := f.listSharedFolders(ctx) - if err != nil { - return "", err - } - for _, entry := range entries { + errFoundFile := errors.New("found file") + err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error { if entry.(*fs.Dir).Remote() == name { - return entry.(*fs.Dir).ID(), nil + id = entry.(*fs.Dir).ID() + return errFoundFile } + return nil + }) + if errors.Is(err, errFoundFile) { + return id, nil + } else if err != nil { + return "", err } return "", fs.ErrorDirNotFound } @@ -908,7 +913,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error { // listReceivedFiles lists shared the user as access to (note this means individual // files not files contained in shared folders) -func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) { +func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) error) (err error) { started := false var res *sharing.ListFilesResult for { @@ -921,7 +926,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err return shouldRetry(ctx, err) }) if err != nil { - return nil, err + return err } started = true } else { @@ -933,7 +938,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err return shouldRetry(ctx, err) }) if err != nil { - return nil, fmt.Errorf("list continue: %w", err) + return fmt.Errorf("list continue: %w", err) } } for _, entry := range res.Entries { @@ -946,26 +951,33 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err modTime: *entry.TimeInvited, } if err != nil { - return nil, err + return err + } + err = callback(o) + if err != nil { + return err } - entries = append(entries, o) } if res.Cursor == "" { break } } - return entries, nil + return nil } func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) { - files, err := f.listReceivedFiles(ctx) - if err != nil { - return nil, err - } - for _, entry := range files { + errFoundFile := errors.New("found file") + err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error { if entry.(*Object).remote == name { - return entry.(*Object), nil + o = entry.(*Object) + return errFoundFile } + return nil + }) + if errors.Is(err, errFoundFile) { + return o, nil + } else if err != nil { + return nil, err } return nil, fs.ErrorObjectNotFound } @@ -980,11 +992,37 @@ func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err er // This should return ErrDirNotFound if the directory isn't // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + return list.WithListP(ctx, dir, f) +} + +// ListP lists the objects and directories of the Fs starting +// from dir non recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { + list := list.NewHelper(callback) if f.opt.SharedFiles { - return f.listReceivedFiles(ctx) + err := f.listReceivedFiles(ctx, list.Add) + if err != nil { + return err + } + return list.Flush() } if f.opt.SharedFolders { - return f.listSharedFolders(ctx) + err := f.listSharedFolders(ctx, list.Add) + if err != nil { + return err + } + return list.Flush() } root := f.slashRoot @@ -1014,7 +1052,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e err = fs.ErrorDirNotFound } } - return nil, err + return err } started = true } else { @@ -1026,7 +1064,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return shouldRetry(ctx, err) }) if err != nil { - return nil, fmt.Errorf("list continue: %w", err) + return fmt.Errorf("list continue: %w", err) } } for _, entry := range res.Entries { @@ -1051,14 +1089,20 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e remote := path.Join(dir, leaf) if folderInfo != nil { d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id) - entries = append(entries, d) + err = list.Add(d) + if err != nil { + return err + } } else if fileInfo != nil { o, err := f.newObjectWithInfo(ctx, remote, fileInfo) if err != nil { - return nil, err + return err } if o.(*Object).exportType.listable() { - entries = append(entries, o) + err = list.Add(o) + if err != nil { + return err + } } } } @@ -1066,7 +1110,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e break } } - return entries, nil + return list.Flush() } // Put the object @@ -2087,6 +2131,7 @@ var ( _ fs.Mover = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) + _ fs.ListPer = (*Fs)(nil) _ fs.Abouter = (*Fs)(nil) _ fs.Shutdowner = &Fs{} _ fs.Object = (*Object)(nil)