jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-01-06 10:33:34 +00:00
Code Issues Projects Releases Wiki Activity

Refactor the List and ListDir interface

Browse Source 
Gives more accurate error propagation, control of depth of recursion
and short circuit recursion where possible.

Most of the the heavy lifting is done in the "fs" package, making file
system implementations a bit simpler.

This commit contains some code originally by Klaus Post.

Fixes #316
This commit is contained in:
Nick Craig-Wood
2016-04-21 20:06:21 +01:00
parent 3bdad260b0
commit 753b0717be
21 changed files with 1512 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fstest/fstest.go
View File

@@ -154,12 +154,13 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
var err error
const retries = 6
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs = nil
for obj := range f.List() {
objs = append(objs, obj)
objs, err = fs.NewLister().Start(f).GetObjects()
if err != nil && err != fs.ErrorDirNotFound {
t.Fatalf("Error listing: %v", err)
}
if len(objs) == len(items) {
// Put an extra sleep in if we did any retries just to make sure it really

42
fstest/fstests/fstests.go
View File

@@ -129,8 +129,12 @@ func TestFsListEmpty(t *testing.T) {
// TestFsListDirEmpty tests listing the directories from an empty directory
func TestFsListDirEmpty(t *testing.T) {
skipIfNotOk(t)
for obj := range remote.ListDir() {
t.Errorf("Found unexpected item %q", obj.Name)
dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs()
if err != nil {
t.Fatal(err)
}
for _, dir := range dirs {
t.Errorf("Found unexpected item %q", dir.Name)
}
}
@@ -193,9 +197,13 @@ func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t)
found := false
for i := 1; i <= eventualConsistencyRetries; i++ {
for obj := range remote.ListDir() {
if obj.Name != `hello? sausage` && obj.Name != `hello_ sausage` {
t.Errorf("Found unexpected item %q", obj.Name)
dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs()
if err != nil {
t.Fatal(err)
}
for _, dir := range dirs {
if dir.Name != `hello? sausage` && dir.Name != `hello_ sausage` {
t.Errorf("Found unexpected item %q", dir.Name)
} else {
found = true
}
@@ -219,8 +227,12 @@ func TestFsListDirRoot(t *testing.T) {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
found := false
for obj := range rootRemote.ListDir() {
if obj.Name == subRemoteLeaf {
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote).GetDirs()
if err != nil {
t.Fatal(err)
}
for _, dir := range dirs {
if dir.Name == subRemoteLeaf {
found = true
}
}
@@ -243,8 +255,11 @@ func TestFsListRoot(t *testing.T) {
f2 := subRemoteLeaf + "/" + file2.Path
f2Alt := subRemoteLeaf + "/" + file2.WinPath
count := 0
errors := fs.Stats.GetErrors()
for obj := range rootRemote.List() {
objs, err := fs.NewLister().Start(rootRemote).GetObjects()
if err != nil {
t.Fatal(err)
}
for _, obj := range objs {
count++
if obj.Remote() == f1 {
found1 = true
@@ -253,17 +268,12 @@ func TestFsListRoot(t *testing.T) {
found2 = true
}
}
errors -= fs.Stats.GetErrors()
if count == 0 {
if errors == 0 {
t.Error("Expecting error if count==0")
}
// Nothing found is OK
return
}
if found1 && found2 {
if errors != 0 {
t.Error("Not expecting error if found")
}
// Both found is OK
return
}
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count)