1
0
mirror of https://github.com/rclone/rclone.git synced 2025-12-06 00:03:32 +00:00

Make --files-from only read the objects specified and don't scan directories

Before this change using --files-from would scan all the directories
that the files could possibly be in causing rclone to do more work
that was necessary.

After this change, rclone constructs an in memory tree using the
--fast-list mechanism but from all of the files in the --files-from
list and without scanning any directories.

Any objects that are not found in the --files-from list are ignored
silently.

This mechanism is used for sync/copy/move (march) and all of the
listing commands ls/lsf/md5sum/etc (walk).
This commit is contained in:
Nick Craig-Wood
2018-10-19 17:41:14 +01:00
parent 9959c5f17f
commit c5ac96e9e7
7 changed files with 179 additions and 1 deletions

View File

@@ -496,3 +496,31 @@ func (f *Filter) DumpFilters() string {
}
return strings.Join(rules, "\n")
}
// HaveFilesFrom returns true if --files-from has been supplied
func (f *Filter) HaveFilesFrom() bool {
return f.files != nil
}
var errFilesFromNotSet = errors.New("--files-from not set so can't use Filter.ListR")
// MakeListR makes function to return all the files set using --files-from
func (f *Filter) MakeListR(NewObject func(remote string) (fs.Object, error)) fs.ListRFn {
return func(dir string, callback fs.ListRCallback) error {
if !f.HaveFilesFrom() {
return errFilesFromNotSet
}
var entries fs.DirEntries
for remote := range f.files {
entry, err := NewObject(remote)
if err == fs.ErrorObjectNotFound {
// Skip files that are not found
} else if err != nil {
return err
} else {
entries = append(entries, entry)
}
}
return callback(entries)
}
}