diff --git a/fs/list/list.go b/fs/list/list.go index edd56f889..d2caf0323 100644 --- a/fs/list/list.go +++ b/fs/list/list.go @@ -39,8 +39,64 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f)) } -// filter (if required) and check the entries, then sort them -func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string, +// listP for every backend +func listP(ctx context.Context, f fs.Fs, dir string, callback fs.ListRCallback) error { + if doListP := f.Features().ListP; doListP != nil { + return doListP(ctx, dir, callback) + } + // Fallback to List + entries, err := f.List(ctx, dir) + if err != nil { + return err + } + return callback(entries) +} + +// DirSortedFn reads Object and *Dir into entries for the given Fs. +// +// dir is the start directory, "" for root +// +// If includeAll is specified all files will be added, otherwise only +// files and directories passing the filter will be added. +// +// Files will be returned through callback in sorted order +func DirSortedFn(ctx context.Context, f fs.Fs, includeAll bool, dir string, callback fs.ListRCallback, keyFn KeyFn) (err error) { + stats := accounting.Stats(ctx) + fi := filter.GetConfig(ctx) + + // Sort the entries, in or out of memory + sorter, err := NewSorter(ctx, callback, keyFn) + if err != nil { + return fmt.Errorf("failed to create directory sorter: %w", err) + } + defer sorter.CleanUp() + + // Get unfiltered entries from the fs + err = listP(ctx, f, dir, func(entries fs.DirEntries) error { + stats.Listed(int64(len(entries))) + + // This should happen only if exclude files lives in the + // starting directory, otherwise ListDirSorted should not be + // called. + if !includeAll && fi.ListContainsExcludeFile(entries) { + fs.Debugf(dir, "Excluded") + return nil + } + + entries, err := filterDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f)) + if err != nil { + return err + } + return sorter.Add(entries) + }) + if err != nil { + return err + } + return sorter.Send() +} + +// Filter the entries passed in +func filterDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string, IncludeObject func(ctx context.Context, o fs.Object) bool, IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) { newEntries = entries[:0] // in place filter @@ -96,7 +152,18 @@ func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll boo newEntries = append(newEntries, entry) } } - entries = newEntries + return newEntries, nil +} + +// filter and sort the entries +func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string, + IncludeObject func(ctx context.Context, o fs.Object) bool, + IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) { + // Filter the directory entries (in place) + entries, err = filterDir(ctx, entries, includeAll, dir, IncludeObject, IncludeDirectory) + if err != nil { + return nil, err + } // Sort the directory entries by Remote // diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go index 1bcd8860e..4719d6c55 100644 --- a/fs/operations/listdirsorted_test.go +++ b/fs/operations/listdirsorted_test.go @@ -12,9 +12,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestListDirSorted is integration testing code in fs/list/list.go +// testListDirSorted is integration testing code in fs/list/list.go // which can't be tested there due to import loops. -func TestListDirSorted(t *testing.T) { +func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) { r := fstest.NewRun(t) ctx := context.Background() @@ -52,20 +52,20 @@ func TestListDirSorted(t *testing.T) { return name } - items, err = list.DirSorted(context.Background(), r.Fremote, true, "") + items, err = listFn(context.Background(), r.Fremote, true, "") require.NoError(t, err) require.Len(t, items, 3) assert.Equal(t, "a.txt", str(0)) assert.Equal(t, "sub dir/", str(1)) assert.Equal(t, "zend.txt", str(2)) - items, err = list.DirSorted(context.Background(), r.Fremote, false, "") + items, err = listFn(context.Background(), r.Fremote, false, "") require.NoError(t, err) require.Len(t, items, 2) assert.Equal(t, "sub dir/", str(0)) assert.Equal(t, "zend.txt", str(1)) - items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir") + items, err = listFn(context.Background(), r.Fremote, true, "sub dir") require.NoError(t, err) require.Len(t, items, 4) assert.Equal(t, "sub dir/hello world", str(0)) @@ -73,7 +73,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/ignore dir/", str(2)) assert.Equal(t, "sub dir/sub sub dir/", str(3)) - items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") + items, err = listFn(context.Background(), r.Fremote, false, "sub dir") require.NoError(t, err) require.Len(t, items, 2) assert.Equal(t, "sub dir/ignore dir/", str(0)) @@ -82,25 +82,45 @@ func TestListDirSorted(t *testing.T) { // testing ignore file fi.Opt.ExcludeFile = []string{".ignore"} - items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") + items, err = listFn(context.Background(), r.Fremote, false, "sub dir") require.NoError(t, err) require.Len(t, items, 1) assert.Equal(t, "sub dir/sub sub dir/", str(0)) - items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") + items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 0) - items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir") + items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 2) assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) fi.Opt.ExcludeFile = nil - items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") + items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 2) assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) } + +// TestListDirSorted is integration testing code in fs/list/list.go +// which can't be tested there due to import loops. +func TestListDirSorted(t *testing.T) { + testListDirSorted(t, list.DirSorted) +} + +// TestListDirSortedFn is integration testing code in fs/list/list.go +// which can't be tested there due to import loops. +func TestListDirSortedFn(t *testing.T) { + listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) { + callback := func(newEntries fs.DirEntries) error { + entries = append(entries, newEntries...) + return nil + } + err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil) + return entries, err + } + testListDirSorted(t, listFn) +}