Version v1.06

* Change log.Printf into fs.Log
  * Re-read metadata on SetModtime

.travis.yml

@@ -3,7 +3,8 @@ language: go
 go:
   - 1.1.2
   - 1.2.2
-  - 1.3
+  - 1.3.3
+  - 1.4
   - tip
 
 script:

Makefile

@@ -8,7 +8,7 @@ rclone:
 
 test:	rclone
 	go test ./...
-	rclonetest/test.sh
+	cd fs && ./test_all.sh
 
 doc:	rclone.1 README.html README.txt
 
@@ -61,3 +61,6 @@ tag:
 
 retag:
 	git tag -f $(LAST_TAG)
+
+gen_tests:
+	cd fstest/fstests && go run gen_tests.go

README.md

@@ -261,6 +261,21 @@ Bugs
 
 Changelog
 ---------
+  * v1.06 - 2014-12-12
+    * Fix "Couldn't find home directory" on OSX
+    * Add tenant parameter for swift
+    * Use new location of Google API packages
+  * v1.05 - 2014-08-09
+    * Improved tests and consequently lots of minor fixes
+    * core: Fix race detected by go race detector
+    * core: Fixes after running errcheck
+    * drive: reset root directory on Rmdir and Purge
+    * fs: Document that Purger returns error on empty directory, test and fix
+    * google cloud storage: fix ListDir on subdirectory
+    * google cloud storage: re-read metadata in SetModTime
+    * s3: make reading metadata more reliable to work around eventual consistency problems
+    * s3: strip trailing / from ListDir()
+    * swift: return directories without / in ListDir
   * v1.04 - 2014-07-21
     * google cloud storage: Fix crash on Update
   * v1.03 - 2014-07-20

docs/content/downloads.md

@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2014-07-21"
+date: "2014-12-12"
 ---
 
-Rclone Download v1.04
+Rclone Download v1.06
 =====================
 
   * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-windows-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-windows-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-windows-amd64.zip)
   * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-osx-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-osx-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-osx-amd64.zip)
   * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.04-linux-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-linux-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-linux-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.06-linux-arm.zip)
   * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.04-freebsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-freebsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-freebsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.06-freebsd-arm.zip)
   * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.04-netbsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-netbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-netbsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.06-netbsd-arm.zip)
   * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.04-openbsd-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-openbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.06-openbsd-amd64.zip)
   * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.04-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.06-plan9-386.zip)
 
 Older downloads can be found [here](http://downloads.rclone.org/)

drive/drive.go

@@ -27,7 +27,8 @@ import (
 	"sync"
 	"time"
 
-	"code.google.com/p/google-api-go-client/drive/v2"
+	"google.golang.org/api/drive/v2"
+
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/googleauth"
 	"github.com/ogier/pflag"
@@ -38,8 +39,8 @@ const (
 	rcloneClientId     = "202264815644.apps.googleusercontent.com"
 	rcloneClientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
 	driveFolderType    = "application/vnd.google-apps.folder"
-	RFC3339In          = time.RFC3339
-	RFC3339Out         = "2006-01-02T15:04:05.000000000Z07:00"
+	timeFormatIn       = time.RFC3339
+	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
 )
 
 // Globals
@@ -234,10 +235,8 @@ func NewFs(name, path string) (fs.Fs, error) {
 		return nil, fmt.Errorf("Couldn't read info about Drive: %s", err)
 	}
 
-	// Find the Id of the root directory and the Id of its parent
-	f.rootId = f.about.RootFolderId
-	// Put the root directory in
-	f.dirCache.Put("", f.rootId)
+	// Find the Id of the true root and clear everything
+	f.resetRoot()
 	// Find the current root
 	err = f.findRoot(false)
 	if err != nil {
@@ -251,7 +250,7 @@ func NewFs(name, path string) (fs.Fs, error) {
 			// No root so return old f
 			return f, nil
 		}
-		obj, err := newF.newFsObjectWithInfo(remote, nil)
+		obj, err := newF.newFsObjectWithInfoErr(remote, nil)
 		if err != nil {
 			// File doesn't exist so return old f
 			return f, nil
@@ -264,7 +263,7 @@ func NewFs(name, path string) (fs.Fs, error) {
 }
 
 // Return an FsObject from a path
-func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
+func (f *FsDrive) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) {
 	fs := &FsObjectDrive{
 		drive:  f,
 		remote: remote,
@@ -284,8 +283,8 @@ func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) (fs.Objec
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
-	fs, _ := f.newFsObjectWithInfo(remote, info)
+func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object {
+	fs, _ := f.newFsObjectWithInfoErr(remote, info)
 	// Errors have already been logged
 	return fs
 }
@@ -294,7 +293,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object
 //
 // May return nil if an error occurred
 func (f *FsDrive) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // Path should be directory path either "" or "path/"
@@ -318,7 +317,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan
 		} else {
 			// If item has no MD5 sum it isn't stored on drive, so ignore it
 			if item.Md5Checksum != "" {
-				if fs := f.NewFsObjectWithInfo(path+item.Title, item); fs != nil {
+				if fs := f.newFsObjectWithInfo(path+item.Title, item); fs != nil {
 					out <- fs
 				}
 			}
@@ -368,7 +367,7 @@ func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) err
 			// fmt.Printf("file %s %s %s\n", path, item.Title, item.Id)
 			// If item has no MD5 sum it isn't stored on drive, so ignore it
 			if item.Md5Checksum != "" {
-				if fs := f.NewFsObjectWithInfo(path, item); fs != nil {
+				if fs := f.newFsObjectWithInfo(path, item); fs != nil {
 					out <- fs
 				}
 			}
@@ -499,7 +498,7 @@ func (f *FsDrive) _findDir(path string, create bool) (pathId string, err error)
 			}
 			info, err := f.svc.Files.Insert(info).Do()
 			if err != nil {
-				return pathId, fmt.Errorf("Failed to make directory")
+				return pathId, fmt.Errorf("Failed to make directory: %v", err)
 			}
 			pathId = info.Id
 		} else {
@@ -537,6 +536,20 @@ func (f *FsDrive) findRoot(create bool) error {
 	return nil
 }
 
+// Resets the root directory to the absolute root and clears the dirCache
+func (f *FsDrive) resetRoot() {
+	f.findRootLock.Lock()
+	defer f.findRootLock.Unlock()
+	f.foundRoot = false
+	f.dirCache.Flush()
+
+	// Put the true root in
+	f.rootId = f.about.RootFolderId
+
+	// Put the root directory in
+	f.dirCache.Put("", f.rootId)
+}
+
 // Walk the path returning a channel of FsObjects
 func (f *FsDrive) List() fs.ObjectsChan {
 	out := make(fs.ObjectsChan, fs.Config.Checkers)
@@ -577,7 +590,7 @@ func (f *FsDrive) ListDir() fs.DirChan {
 					Bytes: -1,
 					Count: -1,
 				}
-				dir.When, _ = time.Parse(RFC3339In, item.ModifiedDate)
+				dir.When, _ = time.Parse(timeFormatIn, item.ModifiedDate)
 				out <- dir
 				return false
 			})
@@ -638,7 +651,7 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
 	if mimeType == "" {
 		mimeType = "application/octet-stream"
 	}
-	modifiedDate := modTime.Format(RFC3339Out)
+	modifiedDate := modTime.Format(timeFormatOut)
 
 	// Define the metadata for the file we are going to create.
 	info := &drive.File{
@@ -686,6 +699,7 @@ func (f *FsDrive) Rmdir() error {
 			return err
 		}
 	}
+	f.resetRoot()
 	return nil
 }
 
@@ -708,7 +722,7 @@ func (f *FsDrive) Purge() error {
 		return err
 	}
 	err = f.svc.Files.Delete(f.rootId).Do()
-	f.dirCache.Flush()
+	f.resetRoot()
 	if err != nil {
 		return err
 	}
@@ -795,7 +809,7 @@ func (o *FsObjectDrive) ModTime() time.Time {
 		fs.Log(o, "Failed to read metadata: %s", err)
 		return time.Now()
 	}
-	modTime, err := time.Parse(RFC3339In, o.modifiedDate)
+	modTime, err := time.Parse(timeFormatIn, o.modifiedDate)
 	if err != nil {
 		fs.Log(o, "Failed to read mtime from object: %s", err)
 		return time.Now()
@@ -813,7 +827,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
 	}
 	// New metadata
 	info := &drive.File{
-		ModifiedDate: modTime.Format(RFC3339Out),
+		ModifiedDate: modTime.Format(timeFormatOut),
 	}
 	// Set modified date
 	_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
@@ -840,7 +854,7 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
 		return nil, err
 	}
 	if res.StatusCode != 200 {
-		res.Body.Close()
+		_ = res.Body.Close() // ignore error
 		return nil, fmt.Errorf("Bad response: %d: %s", res.StatusCode, res.Status)
 	}
 	return res.Body, nil
@@ -854,7 +868,7 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
 func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
 	info := &drive.File{
 		Id:           o.id,
-		ModifiedDate: modTime.Format(RFC3339Out),
+		ModifiedDate: modTime.Format(timeFormatOut),
 	}
 
 	// Make the API request to upload metadata and file data.

drive/drive_test.go

@@ -0,0 +1,53 @@
+// Test Drive filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package drive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*drive.FsObjectDrive)(nil))
+	fstests.RemoteName = "TestDrive:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

dropbox/dropbox.go

@@ -45,8 +45,8 @@ const (
 	md5sumField      = "md5sum"
 	mtimeField       = "mtime"
 	maxCommitRetries = 5
-	RFC3339In        = time.RFC3339
-	RFC3339Out       = "2006-01-02T15:04:05.000000000Z07:00"
+	timeFormatIn     = time.RFC3339
+	timeFormatOut    = "2006-01-02T15:04:05.000000000Z07:00"
 )
 
 // Register with Fs
@@ -54,7 +54,7 @@ func init() {
 	fs.Register(&fs.FsInfo{
 		Name:   "dropbox",
 		NewFs:  NewFs,
-		Config: Config,
+		Config: configHelper,
 		Options: []fs.Option{{
 			Name: "app_key",
 			Help: "Dropbox App Key - leave blank to use rclone's.",
@@ -66,7 +66,7 @@ func init() {
 }
 
 // Configuration helper - called after the user has put in the defaults
-func Config(name string) {
+func configHelper(name string) {
 	// See if already have a token
 	token := fs.ConfigFile.MustValue(name, "token")
 	if token != "" {
@@ -214,7 +214,9 @@ func (f *FsDropbox) openDataStore() {
 }
 
 // Return an FsObject from a path
-func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) {
+//
+// May return nil if an error occurred
+func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
 	o := &FsObjectDropbox{
 		dropbox: f,
 		remote:  remote,
@@ -225,26 +227,17 @@ func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) (fs.
 		err := o.readEntryAndSetMetadata()
 		if err != nil {
 			// logged already fs.Debug("Failed to read info: %s", err)
-			return nil, err
+			return nil
 		}
 	}
-	return o, nil
-}
-
-// Return an FsObject from a path
-//
-// May return nil if an error occurred
-func (f *FsDropbox) NewFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
-	fs, _ := f.newFsObjectWithInfo(remote, info)
-	// Errors have already been logged
-	return fs
+	return o
 }
 
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
 func (f *FsDropbox) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // Strips the root off entry and returns it
@@ -290,7 +283,7 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
 						// ignore directories
 					} else {
 						path := f.stripRoot(entry)
-						out <- f.NewFsObjectWithInfo(path, entry)
+						out <- f.newFsObjectWithInfo(path, entry)
 					}
 				}
 			}
@@ -574,7 +567,7 @@ func metadataKey(path string) string {
 	// NB File system is case insensitive
 	path = strings.ToLower(path)
 	hash := md5.New()
-	hash.Write([]byte(path))
+	_, _ = hash.Write([]byte(path))
 	return fmt.Sprintf("%x", hash.Sum(nil))
 }
 
@@ -625,7 +618,7 @@ func (o *FsObjectDropbox) readMetaData() (err error) {
 			if !ok {
 				fs.Debug(o, "mtime not a string")
 			} else {
-				modTime, err := time.Parse(RFC3339In, mtime)
+				modTime, err := time.Parse(timeFormatIn, mtime)
 				if err != nil {
 					return err
 				}
@@ -635,8 +628,7 @@ func (o *FsObjectDropbox) readMetaData() (err error) {
 	}
 
 	// Last resort
-	o.readEntryAndSetMetadata()
-	return nil
+	return o.readEntryAndSetMetadata()
 }
 
 // ModTime returns the modification time of the object
@@ -672,7 +664,7 @@ func (o *FsObjectDropbox) setModTimeAndMd5sum(modTime time.Time, md5sum string)
 		}
 
 		if !modTime.IsZero() {
-			mtime := modTime.Format(RFC3339Out)
+			mtime := modTime.Format(timeFormatOut)
 			err := record.Set(mtimeField, mtime)
 			if err != nil {
 				return fmt.Errorf("Couldn't set mtime record: %s", err)

dropbox/dropbox_test.go

@@ -0,0 +1,53 @@
+// Test Dropbox filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package dropbox_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/dropbox"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*dropbox.FsObjectDropbox)(nil))
+	fstests.RemoteName = "TestDropbox:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

fs/config.go

@@ -56,11 +56,21 @@ type ConfigInfo struct {
 func configHome() string {
 	// Find users home directory
 	usr, err := user.Current()
-	if err != nil {
-		log.Printf("Couldn't find home directory: %v", err)
-		return ""
+	if err == nil {
+		return usr.HomeDir
 	}
-	return usr.HomeDir
+	// Fall back to reading $HOME - work around user.Current() not
+	// working for cross compiled binaries on OSX.
+	// https://github.com/golang/go/issues/6376
+	home := os.Getenv("HOME")
+	if home != "" {
+		return home
+	}
+	log.Printf("Couldn't find home directory or read HOME environment variable.")
+	log.Printf("Defaulting to storing config in current directory.")
+	log.Printf("Use -config flag to workaround.")
+	log.Printf("Error was: %v", err)
+	return ""
 }
 
 // Loads the config file

fs/fs.go

@@ -20,6 +20,8 @@ const (
 var (
 	// Filesystem registry
 	fsRegistry []*FsInfo
+	// Error returned by NewFs if not found in config file
+	NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
 )
 
 // Filesystem info
@@ -79,9 +81,13 @@ type Fs interface {
 	Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
 
 	// Make the directory (container, bucket)
+	//
+	// Shouldn't return an error if it already exists
 	Mkdir() error
 
 	// Remove the directory (container, bucket) if empty
+	//
+	// Return an error if it doesn't exist or isn't empty
 	Rmdir() error
 
 	// Precision of the ModTimes in this Fs
@@ -131,6 +137,8 @@ type Purger interface {
 	//
 	// Implement this if you have a way of deleting all the files
 	// quicker than just running Remove() on the result of List()
+	//
+	// Return an error if it doesn't exist
 	Purge() error
 }
 
@@ -176,9 +184,10 @@ func Find(name string) (*FsInfo, error) {
 
 // NewFs makes a new Fs object from the path
 //
-// The path is of the form service://path
+// The path is of the form remote:path
 //
-// Services are looked up in the config file
+// Remotes are looked up in the config file.  If the remote isn't
+// found then NotFoundInConfigFile will be returned.
 func NewFs(path string) (Fs, error) {
 	parts := matcher.FindStringSubmatch(path)
 	fsName, configName, fsPath := "local", "local", path
@@ -187,7 +196,7 @@ func NewFs(path string) (Fs, error) {
 		var err error
 		fsName, err = ConfigFile.GetValue(configName, "type")
 		if err != nil {
-			return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
+			return nil, NotFoundInConfigFile
 		}
 	}
 	fs, err := Find(fsName)

fs/operations.go

@@ -4,7 +4,7 @@ package fs
 
 import (
 	"fmt"
-	"log"
+	"io"
 	"sync"
 )
 
@@ -225,12 +225,10 @@ func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
 func DeleteFiles(to_be_deleted ObjectsChan) {
 	var wg sync.WaitGroup
 	wg.Add(Config.Transfers)
-	var fs Fs
 	for i := 0; i < Config.Transfers; i++ {
 		go func() {
 			defer wg.Done()
 			for dst := range to_be_deleted {
-				fs = dst.Fs()
 				if Config.DryRun {
 					Debug(dst, "Not deleting as --dry-run")
 				} else {
@@ -247,8 +245,7 @@ func DeleteFiles(to_be_deleted ObjectsChan) {
 			}
 		}()
 	}
-
-	Log(fs, "Waiting for deletions to finish")
+	Log(nil, "Waiting for deletions to finish")
 	wg.Wait()
 }
 
@@ -434,9 +431,9 @@ func ListFn(f Fs, fn func(Object)) error {
 // Shows size and path
 //
 // Lists in parallel which may get them out of order
-func List(f Fs) error {
+func List(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
-		fmt.Printf("%9d %s\n", o.Size(), o.Remote())
+		fmt.Fprintf(w, "%9d %s\n", o.Size(), o.Remote())
 	})
 }
 
@@ -445,12 +442,12 @@ func List(f Fs) error {
 // Shows size, mod time and path
 //
 // Lists in parallel which may get them out of order
-func ListLong(f Fs) error {
+func ListLong(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		modTime := o.ModTime()
 		Stats.DoneChecking(o)
-		fmt.Printf("%9d %19s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.00000000"), o.Remote())
+		fmt.Fprintf(w, "%9d %s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.000000000"), o.Remote())
 	})
 }
 
@@ -459,7 +456,7 @@ func ListLong(f Fs) error {
 // Produces the same output as the md5sum command
 //
 // Lists in parallel which may get them out of order
-func Md5sum(f Fs) error {
+func Md5sum(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		md5sum, err := o.Md5sum()
@@ -468,14 +465,14 @@ func Md5sum(f Fs) error {
 			Debug(o, "Failed to read MD5: %v", err)
 			md5sum = "UNKNOWN"
 		}
-		fmt.Printf("%32s  %s\n", md5sum, o.Remote())
+		fmt.Fprintf(w, "%32s  %s\n", md5sum, o.Remote())
 	})
 }
 
 // List the directories/buckets/containers in the Fs to stdout
-func ListDir(f Fs) error {
+func ListDir(f Fs, w io.Writer) error {
 	for dir := range f.ListDir() {
-		fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
+		fmt.Fprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
 	}
 	return nil
 }
@@ -508,20 +505,21 @@ func Rmdir(f Fs) error {
 //
 // FIXME doesn't delete local directories
 func Purge(f Fs) error {
+	var err error
 	if purger, ok := f.(Purger); ok {
 		if Config.DryRun {
 			Debug(f, "Not purging as --dry-run set")
 		} else {
-			err := purger.Purge()
-			if err != nil {
-				Stats.Error()
-				return err
-			}
+			err = purger.Purge()
 		}
 	} else {
+		// DeleteFiles and Rmdir observe --dry-run
 		DeleteFiles(f.List())
-		log.Printf("Deleting path")
-		Rmdir(f)
+		err = Rmdir(f)
+	}
+	if err != nil {
+		Stats.Error()
+		return err
 	}
 	return nil
 }

fs/operations_test.go

@@ -0,0 +1,332 @@
+// Test rclone by doing real transactions to a storage provider to and
+// from the local disk
+
+package fs_test
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+
+	// Active file systems
+	_ "github.com/ncw/rclone/drive"
+	_ "github.com/ncw/rclone/dropbox"
+	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/local"
+	_ "github.com/ncw/rclone/s3"
+	_ "github.com/ncw/rclone/swift"
+)
+
+// Globals
+var (
+	localName, remoteName string
+	flocal, fremote       fs.Fs
+	RemoteName            = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
+	SubDir                = flag.Bool("subdir", false, "Set to test with a sub directory")
+	finalise              func()
+)
+
+// Write a file
+func WriteFile(filePath, content string, t time.Time) {
+	// FIXME make directories?
+	filePath = path.Join(localName, filePath)
+	dirPath := path.Dir(filePath)
+	err := os.MkdirAll(dirPath, 0770)
+	if err != nil {
+		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
+	}
+	err = ioutil.WriteFile(filePath, []byte(content), 0600)
+	if err != nil {
+		log.Fatalf("Failed to write file %q: %v", filePath, err)
+	}
+	err = os.Chtimes(filePath, t, t)
+	if err != nil {
+		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
+	}
+}
+
+var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
+var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
+var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
+
+func TestInit(t *testing.T) {
+	fs.LoadConfig()
+	fs.Config.Verbose = false
+	fs.Config.Quiet = true
+	var err error
+	fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
+	if err != nil {
+		t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
+	}
+	t.Logf("Testing with remote %v", fremote)
+
+	localName, err = ioutil.TempDir("", "rclone")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	t.Logf("Testing with local %q", localName)
+	flocal, err = fs.NewFs(localName)
+	if err != nil {
+		t.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+
+}
+func TestCalculateModifyWindow(t *testing.T) {
+	fs.CalculateModifyWindow(fremote, flocal)
+}
+
+func TestMkdir(t *testing.T) {
+	fstest.TestMkdir(t, fremote)
+}
+
+// Check dry run is working
+func TestCopyWithDryRun(t *testing.T) {
+	WriteFile("sub dir/hello world", "hello world", t1)
+
+	fs.Config.DryRun = true
+	err := fs.Sync(fremote, flocal, false)
+	fs.Config.DryRun = false
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, []fstest.Item{})
+}
+
+// Now without dry run
+func TestCopy(t *testing.T) {
+	err := fs.Sync(fremote, flocal, false)
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestLsd(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.ListDir(fremote, &buf)
+	if err != nil {
+		t.Fatalf("ListDir failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "sub dir\n") {
+		t.Fatalf("Result wrong %q", res)
+	}
+}
+
+// Now delete the local file and download it
+func TestCopyAfterDelete(t *testing.T) {
+	err := os.Remove(localName + "/sub dir/hello world")
+	if err != nil {
+		t.Fatalf("Remove failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+	fstest.CheckListing(t, flocal, []fstest.Item{})
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestCopyRedownload(t *testing.T) {
+	err := fs.Sync(flocal, fremote, false)
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+	fstest.CheckListingWithPrecision(t, flocal, items, fremote.Precision())
+	fstest.CheckListing(t, fremote, items)
+
+	// Clean the directory
+	cleanTempDir(t)
+}
+
+func TestSyncAfterChangingModtimeOnly(t *testing.T) {
+	WriteFile("empty space", "", t1)
+
+	err := os.Chtimes(localName+"/empty space", t2, t2)
+	if err != nil {
+		t.Fatalf("Chtimes failed: %v", err)
+	}
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestSyncAfterAddingAFile(t *testing.T) {
+	WriteFile("potato", "------------------------------------------------------------", t3)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
+	WriteFile("potato", "smaller but same date", t3)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+// Sync after changing a file's contents, modtime but not length
+func TestSyncAfterChangingContentsOnly(t *testing.T) {
+	WriteFile("potato", "SMALLER BUT SAME DATE", t2)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+// Sync after removing a file and adding a file --dry-run
+func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
+	WriteFile("potato2", "------------------------------------------------------------", t1)
+	err := os.Remove(localName + "/potato")
+	if err != nil {
+		t.Fatalf("Remove failed: %v", err)
+	}
+	fs.Config.DryRun = true
+	err = fs.Sync(fremote, flocal, true)
+	fs.Config.DryRun = false
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+
+	before := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, before)
+}
+
+// Sync after removing a file and adding a file
+func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestLs(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.List(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "        0 empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "       60 potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestLsLong(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.ListLong(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	m1 := regexp.MustCompile(`(?m)^        0 2011-12-25 12:59:59\.\d{9} empty space$`)
+	if !m1.MatchString(res) {
+		t.Errorf("empty space missing: %q", res)
+	}
+	m2 := regexp.MustCompile(`(?m)^       60 2001-02-03 04:05:06\.\d{9} potato2$`)
+	if !m2.MatchString(res) {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestMd5sum(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.Md5sum(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e  empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "6548b156ea68a4e003e786df99eee76  potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestCheck(t *testing.T) {
+}
+
+// Clean the temporary directory
+func cleanTempDir(t *testing.T) {
+	t.Logf("Cleaning temporary directory: %q", localName)
+	err := os.RemoveAll(localName)
+	if err != nil {
+		t.Logf("Failed to remove %q: %v", localName, err)
+	}
+}
+
+func TestFinalise(t *testing.T) {
+	finalise()
+
+	cleanTempDir(t)
+}

fs/test_all.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+go install
+
+REMOTES="
+TestSwift:
+TestS3:
+TestDrive:
+TestGoogleCloudStorage:
+TestDropbox:
+"
+
+function test_remote {
+    args=$@
+    echo "@go test $args"
+    go test $args || {
+        echo "*** test $args FAILED ***"
+        exit 1
+    }
+}
+
+test_remote
+test_remote --subdir
+for remote in $REMOTES; do
+    test_remote --remote $remote
+    test_remote --remote $remote --subdir
+done
+
+echo "All OK"

fs/version.go

@@ -1,3 +1,3 @@
 package fs
 
-const Version = "v1.04"
+const Version = "v1.06"

fstest/fstest.go

@@ -0,0 +1,225 @@
+// Utilities for testing the fs
+
+package fstest
+
+// FIXME put name of test FS in Fs structure
+
+import (
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// Seed the random number generator
+func init() {
+	rand.Seed(time.Now().UnixNano())
+
+}
+
+// Represents an item for checking
+type Item struct {
+	Path    string
+	Md5sum  string
+	ModTime time.Time
+	Size    int64
+}
+
+// check the mod time to the given precision
+func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
+	dt := modTime.Sub(i.ModTime)
+	if dt >= precision || dt <= -precision {
+		t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", obj.Remote(), dt, precision, modTime, i.ModTime, precision)
+	}
+}
+
+func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
+	if obj == nil {
+		t.Fatalf("Object is nil")
+	}
+	// Check attributes
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		t.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
+	}
+	if i.Md5sum != Md5sum {
+		t.Errorf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
+	}
+	if i.Size != obj.Size() {
+		t.Errorf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
+	}
+	i.CheckModTime(t, obj, obj.ModTime(), precision)
+}
+
+// Represents all items for checking
+type Items struct {
+	byName map[string]*Item
+	items  []Item
+}
+
+// Make an Items
+func NewItems(items []Item) *Items {
+	is := &Items{
+		byName: make(map[string]*Item),
+		items:  items,
+	}
+	// Fill up byName
+	for i := range items {
+		is.byName[items[i].Path] = &items[i]
+	}
+	return is
+}
+
+// Check off an item
+func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
+	i, ok := is.byName[obj.Remote()]
+	if !ok {
+		t.Errorf("Unexpected file %q", obj.Remote())
+	}
+	delete(is.byName, obj.Remote())
+	i.Check(t, obj, precision)
+}
+
+// Check all done
+func (is *Items) Done(t *testing.T) {
+	if len(is.byName) != 0 {
+		for name := range is.byName {
+			log.Printf("Not found %q", name)
+		}
+		t.Errorf("%d objects not found", len(is.byName))
+	}
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
+	is := NewItems(items)
+	for obj := range f.List() {
+		is.Find(t, obj, precision)
+	}
+	is.Done(t)
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListing(t *testing.T, f fs.Fs, items []Item) {
+	precision := f.Precision()
+	CheckListingWithPrecision(t, f, items, precision)
+}
+
+// Parse a time string or explode
+func Time(timeString string) time.Time {
+	t, err := time.Parse(time.RFC3339Nano, timeString)
+	if err != nil {
+		log.Fatalf("Failed to parse time %q: %v", timeString, err)
+	}
+	return t
+}
+
+// Create a random string
+func RandomString(n int) string {
+	source := "abcdefghijklmnopqrstuvwxyz0123456789"
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = source[rand.Intn(len(source))]
+	}
+	return string(out)
+}
+
+// Creates a temporary directory name for local remotes
+func LocalRemote() (path string, err error) {
+	path, err = ioutil.TempDir("", "rclone")
+	if err == nil {
+		// Now remove the directory
+		err = os.Remove(path)
+	}
+	return
+}
+
+// Make a random bucket or subdirectory name
+//
+// Returns a random remote name plus the leaf name
+func RandomRemoteName(remoteName string) (string, string, error) {
+	var err error
+	var leafName string
+
+	// Make a directory if remote name is null
+	if remoteName == "" {
+		remoteName, err = LocalRemote()
+		if err != nil {
+			return "", "", err
+		}
+	} else {
+		if !strings.HasSuffix(remoteName, ":") {
+			remoteName += "/"
+		}
+		leafName = RandomString(32)
+		remoteName += leafName
+	}
+	return remoteName, leafName, nil
+}
+
+// Make a random bucket or subdirectory on the remote
+//
+// Call the finalise function returned to Purge the fs at the end (and
+// the parent if necessary)
+func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
+	var err error
+	var parentRemote fs.Fs
+
+	remoteName, _, err = RandomRemoteName(remoteName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if subdir {
+		parentRemote, err = fs.NewFs(remoteName)
+		if err != nil {
+			return nil, nil, err
+		}
+		remoteName += "/" + RandomString(8)
+	}
+
+	remote, err := fs.NewFs(remoteName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	finalise := func() {
+		_ = fs.Purge(remote) // ignore error
+		if parentRemote != nil {
+			err = fs.Purge(parentRemote) // ignore error
+			if err != nil {
+				log.Printf("Failed to purge %v: %v", parentRemote, err)
+			}
+		}
+	}
+
+	return remote, finalise, nil
+}
+
+func TestMkdir(t *testing.T, remote fs.Fs) {
+	err := fs.Mkdir(remote)
+	if err != nil {
+		t.Fatalf("Mkdir failed: %v", err)
+	}
+	CheckListing(t, remote, []Item{})
+}
+
+func TestPurge(t *testing.T, remote fs.Fs) {
+	err := fs.Purge(remote)
+	if err != nil {
+		t.Fatalf("Purge failed: %v", err)
+	}
+	CheckListing(t, remote, []Item{})
+}
+
+func TestRmdir(t *testing.T, remote fs.Fs) {
+	err := fs.Rmdir(remote)
+	if err != nil {
+		t.Fatalf("Rmdir failed: %v", err)
+	}
+}

fstest/fstests/fstests.go

@@ -0,0 +1,438 @@
+// Generic tests for testing the Fs and Object interfaces
+package fstests
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+)
+
+var (
+	remote        fs.Fs
+	RemoteName    = ""
+	subRemoteName = ""
+	subRemoteLeaf = ""
+	NilObject     fs.Object
+	file1         = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+		Path:    "file name.txt",
+	}
+	file2 = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
+		Path:    `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`,
+	}
+)
+
+func TestInit(t *testing.T) {
+	var err error
+	fs.LoadConfig()
+	fs.Config.Verbose = false
+	fs.Config.Quiet = true
+	if RemoteName == "" {
+		RemoteName, err = fstest.LocalRemote()
+		if err != nil {
+			log.Fatalf("Failed to create tmp dir: %v", err)
+		}
+	}
+	subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
+	if err != nil {
+		t.Fatalf("Couldn't make remote name: %v", err)
+	}
+
+	remote, err = fs.NewFs(subRemoteName)
+	if err == fs.NotFoundInConfigFile {
+		log.Printf("Didn't find %q in config file - skipping tests", RemoteName)
+		return
+	}
+	if err != nil {
+		t.Fatalf("Couldn't start FS: %v", err)
+	}
+	fstest.TestMkdir(t, remote)
+}
+
+func skipIfNotOk(t *testing.T) {
+	if remote == nil {
+		t.Skip("FS not configured")
+	}
+}
+
+// String returns a description of the FS
+
+func TestFsString(t *testing.T) {
+	skipIfNotOk(t)
+	str := remote.String()
+	if str == "" {
+		t.Fatal("Bad fs.String()")
+	}
+}
+
+type TestFile struct {
+	ModTime time.Time
+	Path    string
+	Size    int64
+	Md5sum  string
+}
+
+func TestFsRmdirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestRmdir(t, remote)
+}
+
+func TestFsRmdirNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on Rmdir non existent")
+	}
+}
+
+func TestFsMkdir(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestMkdir(t, remote)
+	fstest.TestMkdir(t, remote)
+}
+
+func TestFsListEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(t, remote, []fstest.Item{})
+}
+
+func TestFsListDirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	for obj := range remote.ListDir() {
+		t.Error("Found unexpected item %q", obj.Name)
+	}
+}
+
+func TestFsNewFsObjectNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	if remote.NewFsObject("potato") != nil {
+		t.Fatal("Didn't expect to find object")
+	}
+}
+
+func findObject(t *testing.T, Name string) fs.Object {
+	obj := remote.NewFsObject(Name)
+	if obj == nil {
+		t.Fatalf("Object not found: %q", Name)
+	}
+	return obj
+}
+
+func testPut(t *testing.T, file *fstest.Item) {
+	buf := bytes.NewBufferString(fstest.RandomString(100))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file.Size = int64(buf.Len())
+	obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
+	if err != nil {
+		t.Fatal("Put error", err)
+	}
+	file.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file.Check(t, obj, remote.Precision())
+	// Re-read the object and check again
+	obj = findObject(t, file.Path)
+	file.Check(t, obj, remote.Precision())
+}
+
+func TestFsPutFile1(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file1)
+}
+
+func TestFsPutFile2(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file2)
+}
+
+func TestFsListDirFile2(t *testing.T) {
+	skipIfNotOk(t)
+	found := false
+	for obj := range remote.ListDir() {
+		if obj.Name != `hello? sausage` {
+			t.Errorf("Found unexpected item %q", obj.Name)
+		} else {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("Didn't find %q", `hello? sausage`)
+	}
+}
+
+func TestFsListDirRoot(t *testing.T) {
+	skipIfNotOk(t)
+	rootRemote, err := fs.NewFs(RemoteName)
+	if err != nil {
+		t.Fatal("Failed to make remote %q: %v", RemoteName, err)
+	}
+	found := false
+	for obj := range rootRemote.ListDir() {
+		if obj.Name == subRemoteLeaf {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("Didn't find %q", subRemoteLeaf)
+	}
+}
+
+func TestFsListRoot(t *testing.T) {
+	skipIfNotOk(t)
+	rootRemote, err := fs.NewFs(RemoteName)
+	if err != nil {
+		t.Fatal("Failed to make remote %q: %v", RemoteName, err)
+	}
+	// Should either find file1 and file2 or nothing
+	found1 := false
+	file1 := subRemoteLeaf + "/" + file1.Path
+	found2 := false
+	file2 := subRemoteLeaf + "/" + file2.Path
+	count := 0
+	errors := fs.Stats.GetErrors()
+	for obj := range rootRemote.List() {
+		count++
+		if obj.Remote() == file1 {
+			found1 = true
+		}
+		if obj.Remote() == file2 {
+			found2 = true
+		}
+	}
+	errors -= fs.Stats.GetErrors()
+	if count == 0 {
+		if errors == 0 {
+			t.Error("Expecting error if count==0")
+		}
+		return
+	}
+	if found1 && found2 {
+		if errors != 0 {
+			t.Error("Not expecting error if found")
+		}
+		return
+	}
+	t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", file1, found1, file2, found2, count)
+}
+
+func TestFsListFile1(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
+}
+
+func TestFsNewFsObject(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	file1.Check(t, obj, remote.Precision())
+}
+
+func TestFsListFile1and2(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
+}
+
+func TestFsRmdirFull(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on RMdir on non empty remote")
+	}
+}
+
+func TestFsPrecision(t *testing.T) {
+	skipIfNotOk(t)
+	precision := remote.Precision()
+	if precision > time.Second || precision < 0 {
+		t.Fatalf("Precision out of range %v", precision)
+	}
+	// FIXME check expected precision
+}
+
+func TestObjectString(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	s := obj.String()
+	if s != file1.Path {
+		t.Errorf("String() wrong %v != %v", s, file1.Path)
+	}
+	obj = NilObject
+	s = obj.String()
+	if s != "<nil>" {
+		t.Errorf("String() wrong %v != %v", s, "<nil>")
+	}
+}
+
+func TestObjectFs(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Fs() != remote {
+		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
+	}
+}
+
+func TestObjectRemote(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Remote() != file1.Path {
+		t.Errorf("Remote is wrong %v != %v", obj.Remote(), file1.Path)
+	}
+}
+
+func TestObjectMd5sum(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		t.Errorf("Error in Md5sum: %v", err)
+	}
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectModTime(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
+}
+
+func TestObjectSetModTime(t *testing.T) {
+	skipIfNotOk(t)
+	newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
+	obj := findObject(t, file1.Path)
+	obj.SetModTime(newModTime)
+	file1.ModTime = newModTime
+	file1.CheckModTime(t, obj, newModTime, remote.Precision())
+	// And make a new object and read it from there too
+	TestObjectModTime(t)
+}
+
+func TestObjectSize(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Size() != file1.Size {
+		t.Errorf("Size is wrong %v != %v", obj.Size(), file1.Size)
+	}
+}
+
+func TestObjectOpen(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	in, err := obj.Open()
+	if err != nil {
+		t.Fatalf("Open() return error: %v", err)
+	}
+	hash := md5.New()
+	n, err := io.Copy(hash, in)
+	if err != nil {
+		t.Fatalf("io.Copy() return error: %v", err)
+	}
+	if n != file1.Size {
+		t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
+	}
+	err = in.Close()
+	if err != nil {
+		t.Fatalf("in.Close() return error: %v", err)
+	}
+	Md5sum := hex.EncodeToString(hash.Sum(nil))
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectUpdate(t *testing.T) {
+	skipIfNotOk(t)
+	buf := bytes.NewBufferString(fstest.RandomString(200))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file1.Size = int64(buf.Len())
+	obj := findObject(t, file1.Path)
+	err := obj.Update(in, file1.ModTime, file1.Size)
+	if err != nil {
+		t.Fatal("Update error", err)
+	}
+	file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file1.Check(t, obj, remote.Precision())
+	// Re-read the object and check again
+	obj = findObject(t, file1.Path)
+	file1.Check(t, obj, remote.Precision())
+}
+
+func TestObjectStorable(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if !obj.Storable() {
+		t.Fatalf("Expecting %v to be storable", obj)
+	}
+}
+
+func TestLimitedFs(t *testing.T) {
+	skipIfNotOk(t)
+	remoteName := subRemoteName + "/" + file2.Path
+	file2Copy := file2
+	file2Copy.Path = "z.txt"
+	fileRemote, err := fs.NewFs(remoteName)
+	if err != nil {
+		t.Fatal("Failed to make remote %q: %v", remoteName, err)
+	}
+	fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
+	_, ok := fileRemote.(*fs.Limited)
+	if !ok {
+		t.Errorf("%v is not a fs.Limited", fileRemote)
+	}
+}
+
+func TestLimitedFsNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	remoteName := subRemoteName + "/not found.txt"
+	fileRemote, err := fs.NewFs(remoteName)
+	if err != nil {
+		t.Fatal("Failed to make remote %q: %v", remoteName, err)
+	}
+	fstest.CheckListing(t, fileRemote, []fstest.Item{})
+	_, ok := fileRemote.(*fs.Limited)
+	if ok {
+		t.Errorf("%v is is a fs.Limited", fileRemote)
+	}
+}
+
+func TestObjectRemove(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	err := obj.Remove()
+	if err != nil {
+		t.Fatal("Remove error", err)
+	}
+	fstest.CheckListing(t, remote, []fstest.Item{file2})
+}
+
+func TestObjectPurge(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestPurge(t, remote)
+	err := fs.Purge(remote)
+	if err == nil {
+		t.Fatal("Expecting error after on second purge")
+	}
+}
+
+func TestFinalise(t *testing.T) {
+	skipIfNotOk(t)
+	if strings.HasPrefix(RemoteName, "/") {
+		// Remove temp directory
+		err := os.Remove(RemoteName)
+		if err != nil {
+			log.Printf("Failed to remove %q: %v\n", RemoteName, err)
+		}
+	}
+}

fstest/fstests/gen_tests.go

@@ -0,0 +1,143 @@
+// +build ignore
+
+// Make the test files from fstests.go
+package main
+
+import (
+	"bufio"
+	"html/template"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// Search fstests.go and return all the test function names
+func findTestFunctions() []string {
+	fns := []string{}
+	matcher := regexp.MustCompile(`^func\s+(Test.*?)\(`)
+
+	in, err := os.Open("fstests.go")
+	if err != nil {
+		log.Fatalf("Couldn't open fstests.go: %v", err)
+	}
+	defer in.Close()
+
+	scanner := bufio.NewScanner(in)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := matcher.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			fns = append(fns, matches[1])
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatalf("Error scanning file: %v", err)
+	}
+	return fns
+}
+
+// Data to substitute
+type Data struct {
+	Regenerate  string
+	FsName      string
+	UpperFsName string
+	TestName    string
+	ObjectName  string
+	Fns         []string
+}
+
+var testProgram = `
+// Test {{ .UpperFsName }} filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: {{ .Regenerate }}
+package {{ .FsName }}_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/{{ .FsName }}"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*{{ .FsName }}.FsObject{{ .ObjectName }})(nil))
+	fstests.RemoteName = "{{ .TestName }}"
+}
+
+// Generic tests for the Fs
+{{ range $fn := .Fns }}func {{ $fn }}(t *testing.T){ fstests.{{ $fn }}(t) }
+{{ end }}
+`
+
+// Generate test file piping it through gofmt
+func generateTestProgram(t *template.Template, fns []string, Fsname string) {
+	fsname := strings.ToLower(Fsname)
+	TestName := "Test" + Fsname + ":"
+	outfile := "../../" + fsname + "/" + fsname + "_test.go"
+	// Find last capitalised group to be object name
+	matcher := regexp.MustCompile(`([A-Z][a-z0-9]+)$`)
+	matches := matcher.FindStringSubmatch(Fsname)
+	if len(matches) == 0 {
+		log.Fatalf("Couldn't find object name in %q", Fsname)
+	}
+	ObjectName := matches[1]
+
+	if fsname == "local" {
+		TestName = ""
+	}
+
+	data := Data{
+		Regenerate:  "go run gen_tests.go or make gen_tests",
+		FsName:      fsname,
+		UpperFsName: Fsname,
+		TestName:    TestName,
+		ObjectName:  ObjectName,
+		Fns:         fns,
+	}
+
+	cmd := exec.Command("gofmt")
+
+	log.Printf("Writing %q", outfile)
+	out, err := os.Create(outfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	cmd.Stdout = out
+
+	gofmt, err := cmd.StdinPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err = cmd.Start(); err != nil {
+		log.Fatal(err)
+	}
+	if err = t.Execute(gofmt, data); err != nil {
+		log.Fatal(err)
+	}
+	if err = gofmt.Close(); err != nil {
+		log.Fatal(err)
+	}
+	if err = cmd.Wait(); err != nil {
+		log.Fatal(err)
+	}
+	if err = out.Close(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func main() {
+	fns := findTestFunctions()
+	t := template.Must(template.New("main").Parse(testProgram))
+	generateTestProgram(t, fns, "Local")
+	generateTestProgram(t, fns, "Swift")
+	generateTestProgram(t, fns, "S3")
+	generateTestProgram(t, fns, "Drive")
+	generateTestProgram(t, fns, "GoogleCloudStorage")
+	generateTestProgram(t, fns, "Dropbox")
+	log.Printf("Done")
+}

googlecloudstorage/googlecloudstorage.go

@@ -24,8 +24,8 @@ import (
 	"strings"
 	"time"
 
-	"code.google.com/p/google-api-go-client/googleapi"
-	"code.google.com/p/google-api-go-client/storage/v1"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/storage/v1"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/googleauth"
@@ -34,8 +34,8 @@ import (
 const (
 	rcloneClientId     = "202264815644.apps.googleusercontent.com"
 	rcloneClientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
-	RFC3339In          = time.RFC3339
-	RFC3339Out         = "2006-01-02T15:04:05.000000000Z07:00"
+	timeFormatIn       = time.RFC3339
+	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
 	metaMtime          = "mtime" // key to store mtime under in metadata
 	listChunks         = 256     // chunk size to read directory listings
 )
@@ -215,7 +215,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsStorage) NewFsObjectWithInfo(remote string, info *storage.Object) fs.Object {
+func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.Object {
 	o := &FsObjectStorage{
 		storage: f,
 		remote:  remote,
@@ -236,7 +236,7 @@ func (f *FsStorage) NewFsObjectWithInfo(remote string, info *storage.Object) fs.
 //
 // May return nil if an error occurred
 func (f *FsStorage) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -255,16 +255,23 @@ func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) {
 			fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
 			return
 		}
-		for _, object := range objects.Items {
-			if directories && !strings.HasSuffix(object.Name, "/") {
-				continue
+		if !directories {
+			for _, object := range objects.Items {
+				if !strings.HasPrefix(object.Name, f.root) {
+					fs.Log(f, "Odd name received %q", object.Name)
+					continue
+				}
+				remote := object.Name[rootLength:]
+				fn(remote, object)
 			}
-			if !strings.HasPrefix(object.Name, f.root) {
-				fs.Log(f, "Odd name received %q", object.Name)
-				continue
+		} else {
+			var object storage.Object
+			for _, prefix := range objects.Prefixes {
+				if !strings.HasSuffix(prefix, "/") {
+					continue
+				}
+				fn(prefix[:len(prefix)-1], &object)
 			}
-			remote := object.Name[rootLength:]
-			fn(remote, object)
 		}
 		if objects.NextPageToken == "" {
 			break
@@ -286,7 +293,7 @@ func (f *FsStorage) List() fs.ObjectsChan {
 		go func() {
 			defer close(out)
 			f.list(false, func(remote string, object *storage.Object) {
-				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
+				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})
@@ -434,7 +441,7 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
 	// read mtime out of metadata if available
 	mtimeString, ok := info.Metadata[metaMtime]
 	if ok {
-		modTime, err := time.Parse(RFC3339In, mtimeString)
+		modTime, err := time.Parse(timeFormatIn, mtimeString)
 		if err == nil {
 			o.modTime = modTime
 			return
@@ -444,7 +451,7 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
 	}
 
 	// Fallback to the Updated time
-	modTime, err := time.Parse(RFC3339In, info.Updated)
+	modTime, err := time.Parse(timeFormatIn, info.Updated)
 	if err != nil {
 		fs.Log(o, "Bad time decode: %v", err)
 	} else {
@@ -484,7 +491,7 @@ func (o *FsObjectStorage) ModTime() time.Time {
 // Returns metadata for an object
 func metadataFromModTime(modTime time.Time) map[string]string {
 	metadata := make(map[string]string, 1)
-	metadata[metaMtime] = modTime.Format(RFC3339Out)
+	metadata[metaMtime] = modTime.Format(timeFormatOut)
 	return metadata
 }
 
@@ -496,11 +503,12 @@ func (o *FsObjectStorage) SetModTime(modTime time.Time) {
 		Name:     o.storage.root + o.remote,
 		Metadata: metadataFromModTime(modTime),
 	}
-	_, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do()
+	newObject, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do()
 	if err != nil {
 		fs.Stats.Error()
 		fs.Log(o, "Failed to update remote mtime: %s", err)
 	}
+	o.setMetaData(newObject)
 }
 
 // Is this object storable
@@ -530,7 +538,7 @@ func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) {
 		return nil, err
 	}
 	if res.StatusCode != 200 {
-		res.Body.Close()
+		_ = res.Body.Close() // ignore error
 		return nil, fmt.Errorf("Bad response: %d: %s", res.StatusCode, res.Status)
 	}
 	return res.Body, nil
@@ -551,7 +559,7 @@ func (o *FsObjectStorage) Update(in io.Reader, modTime time.Time, size int64) er
 		Name:        o.storage.root + o.remote,
 		ContentType: contentType,
 		Size:        uint64(size),
-		Updated:     modTime.Format(RFC3339Out), // Doesn't get set
+		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
 	newObject, err := o.storage.svc.Objects.Insert(o.storage.bucket, &object).Media(in).Name(object.Name).PredefinedAcl(o.storage.objectAcl).Do()

googlecloudstorage/googlecloudstorage_test.go

@@ -0,0 +1,53 @@
+// Test GoogleCloudStorage filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package googlecloudstorage_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/googlecloudstorage"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*googlecloudstorage.FsObjectStorage)(nil))
+	fstests.RemoteName = "TestGoogleCloudStorage:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

local/local.go

@@ -8,7 +8,6 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -69,7 +68,7 @@ func (f *FsLocal) String() string {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
+func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
 	path := filepath.Join(f.root, remote)
 	o := &FsObjectLocal{local: f, remote: remote, path: path}
 	if info != nil {
@@ -88,7 +87,7 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object
 //
 // May return nil if an error occurred
 func (f *FsLocal) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // List the path returning a channel of FsObjects
@@ -100,19 +99,19 @@ func (f *FsLocal) List() fs.ObjectsChan {
 		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
 			if err != nil {
 				fs.Stats.Error()
-				log.Printf("Failed to open directory: %s: %s", path, err)
+				fs.Log(f, "Failed to open directory: %s: %s", path, err)
 			} else {
 				remote, err := filepath.Rel(f.root, path)
 				if err != nil {
 					fs.Stats.Error()
-					log.Printf("Failed to get relative path %s: %s", path, err)
+					fs.Log(f, "Failed to get relative path %s: %s", path, err)
 					return nil
 				}
 				if remote == "." {
 					return nil
 					// remote = ""
 				}
-				if fs := f.NewFsObjectWithInfo(remote, fi); fs != nil {
+				if fs := f.newFsObjectWithInfo(remote, fi); fs != nil {
 					if fs.Storable() {
 						out <- fs
 					}
@@ -122,7 +121,7 @@ func (f *FsLocal) List() fs.ObjectsChan {
 		})
 		if err != nil {
 			fs.Stats.Error()
-			log.Printf("Failed to open directory: %s: %s", f.root, err)
+			fs.Log(f, "Failed to open directory: %s: %s", f.root, err)
 		}
 		close(out)
 	}()
@@ -137,7 +136,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 		items, err := ioutil.ReadDir(f.root)
 		if err != nil {
 			fs.Stats.Error()
-			log.Printf("Couldn't find read directory: %s", err)
+			fs.Log(f, "Couldn't find read directory: %s", err)
 		} else {
 			for _, item := range items {
 				if item.IsDir() {
@@ -152,7 +151,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
 						if err != nil {
 							fs.Stats.Error()
-							log.Printf("Failed to open directory: %s: %s", path, err)
+							fs.Log(f, "Failed to open directory: %s: %s", path, err)
 						} else {
 							dir.Count += 1
 							dir.Bytes += fi.Size()
@@ -161,7 +160,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 					})
 					if err != nil {
 						fs.Stats.Error()
-						log.Printf("Failed to open directory: %s: %s", dirpath, err)
+						fs.Log(f, "Failed to open directory: %s: %s", dirpath, err)
 					}
 					out <- dir
 				}
@@ -218,12 +217,15 @@ func (f *FsLocal) readPrecision() (precision time.Duration) {
 	}
 	path := fd.Name()
 	// fmt.Println("Created temp file", path)
-	fd.Close()
+	err = fd.Close()
+	if err != nil {
+		return time.Second
+	}
 
 	// Delete it on return
 	defer func() {
 		// fmt.Println("Remove temp file")
-		os.Remove(path)
+		_ = os.Remove(path) // ignore error
 	}()
 
 	// Find the minimum duration we can detect
@@ -259,6 +261,13 @@ func (f *FsLocal) readPrecision() (precision time.Duration) {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *FsLocal) Purge() error {
+	fi, err := os.Lstat(f.root)
+	if err != nil {
+		return err
+	}
+	if !fi.Mode().IsDir() {
+		return fmt.Errorf("Can't Purge non directory: %q", f.root)
+	}
 	return os.RemoveAll(f.root)
 }
 
@@ -325,6 +334,13 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
 	err := os.Chtimes(o.path, modTime, modTime)
 	if err != nil {
 		fs.Debug(o, "Failed to set mtime on file: %s", err)
+		return
+	}
+	// Re-read metadata
+	err = o.lstat()
+	if err != nil {
+		fs.Debug(o, "Failed to stat: %s", err)
+		return
 	}
 }
 
@@ -335,7 +351,7 @@ func (o *FsObjectLocal) Storable() bool {
 		fs.Debug(o, "Can't transfer non file/directory")
 		return false
 	} else if mode&os.ModeDir != 0 {
-		fs.Debug(o, "FIXME Skipping directory")
+		// fs.Debug(o, "Skipping directory")
 		return false
 	}
 	return true

local/local_test.go

@@ -0,0 +1,53 @@
+// Test Local filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package local_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/local"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*local.FsObjectLocal)(nil))
+	fstests.RemoteName = ""
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

notes.txt

@@ -1,7 +1,16 @@
-Remove FIXME skipping directory
-
 Change lsd command so it doesn't show -1
   * Make sure all Fses show -1 for objects Zero for dates etc
+  * Make test?
+
+Put the TestRemote names into the Fs description
+Make test_all.sh use the TestRemote name automatically
+
+Run errcheck and go vet in the make file
+.. Also race detector?
+
+Get rid of Storable?
+
+Write developer manual
 
 Todo
   * FIXME: More -dry-run checks for object transfer

rclone.go

@@ -96,7 +96,7 @@ var Commands = []Command{
 		Help: `
         List all the objects in the the path with size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.List(fdst)
+			err := fs.List(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}
@@ -110,7 +110,7 @@ var Commands = []Command{
 		Help: `
         List all directories/containers/buckets in the the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListDir(fdst)
+			err := fs.ListDir(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to listdir: %v", err)
 			}
@@ -124,7 +124,7 @@ var Commands = []Command{
 		Help: `
         List all the objects in the the path with modification time, size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListLong(fdst)
+			err := fs.ListLong(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list long: %v", err)
 			}
@@ -138,7 +138,7 @@ var Commands = []Command{
 		Help: `
         Produces an md5sum file for all the objects in the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.Md5sum(fdst)
+			err := fs.Md5sum(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}
@@ -265,7 +265,11 @@ func ParseFlags() {
 			fs.Stats.Error()
 			log.Fatal(err)
 		}
-		pprof.StartCPUProfile(f)
+		err = pprof.StartCPUProfile(f)
+		if err != nil {
+			fs.Stats.Error()
+			log.Fatal(err)
+		}
 		defer pprof.StopCPUProfile()
 	}
 }

rclonetest/rclonetest.go

@@ -1,434 +0,0 @@
-// Test rclone by doing real transactions to a storage provider to and
-// from the local disk
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ogier/pflag"
-
-	// Active file systems
-	_ "github.com/ncw/rclone/drive"
-	_ "github.com/ncw/rclone/dropbox"
-	_ "github.com/ncw/rclone/googlecloudstorage"
-	_ "github.com/ncw/rclone/local"
-	_ "github.com/ncw/rclone/s3"
-	_ "github.com/ncw/rclone/swift"
-)
-
-// Globals
-var (
-	localName, remoteName string
-	version               = pflag.BoolP("version", "V", false, "Print the version number")
-	subDir                = pflag.BoolP("subdir", "S", false, "Test with a sub directory")
-)
-
-// Represents an item for checking
-type Item struct {
-	Path    string
-	Md5sum  string
-	ModTime time.Time
-	Size    int64
-}
-
-// Represents all items for checking
-type Items struct {
-	byName map[string]*Item
-	items  []Item
-}
-
-// Make an Items
-func NewItems(items []Item) *Items {
-	is := &Items{
-		byName: make(map[string]*Item),
-		items:  items,
-	}
-	// Fill up byName
-	for i := range items {
-		is.byName[items[i].Path] = &items[i]
-	}
-	return is
-}
-
-// Check off an item
-func (is *Items) Find(obj fs.Object) {
-	i, ok := is.byName[obj.Remote()]
-	if !ok {
-		log.Fatalf("Unexpected file %q", obj.Remote())
-	}
-	delete(is.byName, obj.Remote())
-	// Check attributes
-	Md5sum, err := obj.Md5sum()
-	if err != nil {
-		log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
-	}
-	if i.Md5sum != Md5sum {
-		log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
-	}
-	if i.Size != obj.Size() {
-		log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
-	}
-	// check the mod time to the given precision
-	modTime := obj.ModTime()
-	dt := modTime.Sub(i.ModTime)
-	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
-		log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
-	}
-
-}
-
-// Check all done
-func (is *Items) Done() {
-	if len(is.byName) != 0 {
-		for name := range is.byName {
-			log.Printf("Not found %q", name)
-		}
-		log.Fatalf("%d objects not found", len(is.byName))
-	}
-}
-
-// Checks the fs to see if it has the expected contents
-func CheckListing(f fs.Fs, items []Item) {
-	is := NewItems(items)
-	for obj := range f.List() {
-		is.Find(obj)
-	}
-	is.Done()
-}
-
-// Parse a time string or explode
-func Time(timeString string) time.Time {
-	t, err := time.Parse(time.RFC3339Nano, timeString)
-	if err != nil {
-		log.Fatalf("Failed to parse time %q: %v", timeString, err)
-	}
-	return t
-}
-
-// Write a file
-func WriteFile(filePath, content string, t time.Time) {
-	// FIXME make directories?
-	filePath = path.Join(localName, filePath)
-	dirPath := path.Dir(filePath)
-	err := os.MkdirAll(dirPath, 0770)
-	if err != nil {
-		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
-	}
-	err = ioutil.WriteFile(filePath, []byte(content), 0600)
-	if err != nil {
-		log.Fatalf("Failed to write file %q: %v", filePath, err)
-	}
-	err = os.Chtimes(filePath, t, t)
-	if err != nil {
-		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
-	}
-}
-
-// Create a random string
-func RandomString(n int) string {
-	source := "abcdefghijklmnopqrstuvwxyz0123456789"
-	out := make([]byte, n)
-	for i := range out {
-		out[i] = source[rand.Intn(len(source))]
-	}
-	return string(out)
-}
-
-func TestMkdir(flocal, fremote fs.Fs) {
-	err := fs.Mkdir(fremote)
-	if err != nil {
-		log.Fatalf("Mkdir failed: %v", err)
-	}
-	items := []Item{}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-}
-
-var t1 = Time("2001-02-03T04:05:06.499999999Z")
-var t2 = Time("2011-12-25T12:59:59.123456789Z")
-var t3 = Time("2011-12-30T12:59:59.000000000Z")
-
-func TestCopy(flocal, fremote fs.Fs) {
-	WriteFile("sub dir/hello world", "hello world", t1)
-
-	// Check dry run is working
-	log.Printf("Copy with --dry-run")
-	fs.Config.DryRun = true
-	err := fs.Sync(fremote, flocal, false)
-	fs.Config.DryRun = false
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	items := []Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-
-	CheckListing(flocal, items)
-	CheckListing(fremote, []Item{})
-
-	// Now without dry run
-
-	log.Printf("Copy")
-	err = fs.Sync(fremote, flocal, false)
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// Now delete the local file and download it
-
-	err = os.Remove(localName + "/sub dir/hello world")
-	if err != nil {
-		log.Fatalf("Remove failed: %v", err)
-	}
-
-	CheckListing(flocal, []Item{})
-	CheckListing(fremote, items)
-
-	log.Printf("Copy - redownload")
-	err = fs.Sync(flocal, fremote, false)
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// Clean the directory
-	cleanTempDir()
-}
-
-func TestSync(flocal, fremote fs.Fs) {
-	WriteFile("empty space", "", t1)
-
-	log.Printf("Sync after changing file modtime only")
-	err := os.Chtimes(localName+"/empty space", t2, t2)
-	if err != nil {
-		log.Fatalf("Chtimes failed: %v", err)
-	}
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items := []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after adding a file")
-	WriteFile("potato", "------------------------------------------------------------", t3)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after changing a file's size only")
-	WriteFile("potato", "smaller but same date", t3)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after changing a file's contents, modtime but not length")
-	WriteFile("potato", "SMALLER BUT SAME DATE", t2)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after removing a file and adding a file --dry-run")
-	WriteFile("potato2", "------------------------------------------------------------", t1)
-	err = os.Remove(localName + "/potato")
-	if err != nil {
-		log.Fatalf("Remove failed: %v", err)
-	}
-	fs.Config.DryRun = true
-	err = fs.Sync(fremote, flocal, true)
-	fs.Config.DryRun = false
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-
-	before := []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	items = []Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, before)
-
-	log.Printf("Sync after removing a file and adding a file")
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	CheckListing(flocal, items)
-	CheckListing(fremote, items)
-}
-
-func TestLs(flocal, fremote fs.Fs) {
-	// Underlying List has been tested above, so we just make sure it runs
-	err := fs.List(fremote)
-	if err != nil {
-		log.Fatalf("List failed: %v", err)
-	}
-}
-
-func TestLsd(flocal, fremote fs.Fs) {
-}
-
-func TestCheck(flocal, fremote fs.Fs) {
-}
-
-func TestPurge(fremote fs.Fs) {
-	err := fs.Purge(fremote)
-	if err != nil {
-		log.Fatalf("Purge failed: %v", err)
-	}
-	unexpected := 0
-	for obj := range fremote.List() {
-		unexpected++
-		log.Printf("Found unexpected item %s", obj.Remote())
-	}
-	if unexpected != 0 {
-		log.Fatalf("exiting as found %d unexpected items", unexpected)
-	}
-}
-
-func TestRmdir(flocal, fremote fs.Fs) {
-	err := fs.Rmdir(fremote)
-	if err != nil {
-		log.Fatalf("Rmdir failed: %v", err)
-	}
-}
-
-func syntaxError() {
-	fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s.
-
-Syntax: [options] remote:
-
-Need a remote: as argument.  This will create a random container or
-directory under it and perform tests on it, deleting it at the end.
-
-Options:
-
-`, fs.Version)
-	pflag.PrintDefaults()
-}
-
-// Clean the temporary directory
-func cleanTempDir() {
-	log.Printf("Cleaning temporary directory: %q", localName)
-	err := os.RemoveAll(localName)
-	if err != nil {
-		log.Printf("Failed to remove %q: %v", localName, err)
-	}
-}
-
-func main() {
-	pflag.Usage = syntaxError
-	pflag.Parse()
-	if *version {
-		fmt.Printf("rclonetest %s\n", fs.Version)
-		os.Exit(0)
-	}
-	fs.LoadConfig()
-	rand.Seed(time.Now().UnixNano())
-	args := pflag.Args()
-
-	if len(args) != 1 {
-		syntaxError()
-		os.Exit(1)
-	}
-
-	remoteName = args[0]
-	if !strings.HasSuffix(remoteName, ":") {
-		remoteName += "/"
-	}
-	remoteName += RandomString(32)
-	var parentRemote fs.Fs
-	if *subDir {
-		var err error
-		parentRemote, err = fs.NewFs(remoteName)
-		if err != nil {
-			log.Fatalf("Failed to make parent %q: %v", remoteName, err)
-		}
-		remoteName += "/" + RandomString(8)
-	}
-	log.Printf("Testing with remote %q", remoteName)
-	var err error
-	localName, err = ioutil.TempDir("", "rclone")
-	if err != nil {
-		log.Fatalf("Failed to create temp dir: %v", err)
-	}
-	log.Printf("Testing with local %q", localName)
-
-	fremote, err := fs.NewFs(remoteName)
-	if err != nil {
-		log.Fatalf("Failed to make %q: %v", remoteName, err)
-	}
-	flocal, err := fs.NewFs(localName)
-	if err != nil {
-		log.Fatalf("Failed to make %q: %v", remoteName, err)
-	}
-
-	fs.CalculateModifyWindow(fremote, flocal)
-
-	TestMkdir(flocal, fremote)
-	TestCopy(flocal, fremote)
-	TestSync(flocal, fremote)
-	TestLs(flocal, fremote)
-	TestLsd(flocal, fremote)
-	TestCheck(flocal, fremote)
-	TestPurge(fremote)
-	//TestRmdir(flocal, fremote)
-
-	if parentRemote != nil {
-		TestPurge(parentRemote)
-	}
-
-	cleanTempDir()
-	log.Printf("Tests OK")
-}

rclonetest/test.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-go install
-
-REMOTES="
-memstore:
-s3:
-drive2:
-gcs:
-dropbox:
-/tmp/z
-"
-
-function test_remote {
-    args=$@
-    rclonetest $args || {
-        echo "*** rclonetest $args FAILED ***"
-        exit 1
-    }
-}
-
-for remote in $REMOTES; do
-    test_remote $remote
-    test_remote --subdir $remote
-done

s3/s3.go

@@ -227,7 +227,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
+func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
 	o := &FsObjectS3{
 		s3:     f,
 		remote: remote,
@@ -256,7 +256,7 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
 //
 // May return nil if an error occurred
 func (f *FsS3) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -281,6 +281,9 @@ func (f *FsS3) list(directories bool, fn func(string, *s3.Key)) {
 					continue
 				}
 				remote := remote[rootLength:]
+				if strings.HasSuffix(remote, "/") {
+					remote = remote[:len(remote)-1]
+				}
 				fn(remote, &s3.Key{Key: remote})
 			}
 		} else {
@@ -309,7 +312,7 @@ func (f *FsS3) List() fs.ObjectsChan {
 		go func() {
 			defer close(out)
 			f.list(false, func(remote string, object *s3.Key) {
-				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
+				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})
@@ -418,13 +421,28 @@ func (o *FsObjectS3) Size() int64 {
 
 // readMetaData gets the metadata if it hasn't already been fetched
 //
+// if we get a 404 error then we retry a few times for eventual
+// consistency reasons
+//
 // it also sets the info
 func (o *FsObjectS3) readMetaData() (err error) {
 	if o.meta != nil {
 		return nil
 	}
+	var headers s3.Headers
 
-	headers, err := o.s3.b.Head(o.s3.root+o.remote, nil)
+	// Try reading the metadata a few times (with exponential
+	// backoff) to get around eventual consistency on 404 error
+	for tries := uint(0); tries < 10; tries++ {
+		headers, err = o.s3.b.Head(o.s3.root+o.remote, nil)
+		if s3Err, ok := err.(*s3.Error); ok {
+			if s3Err.StatusCode == http.StatusNotFound {
+				time.Sleep(5 * time.Millisecond << tries)
+				continue
+			}
+		}
+		break
+	}
 	if err != nil {
 		fs.Debug(o, "Failed to read info: %s", err)
 		return err

s3/s3_test.go

@@ -0,0 +1,53 @@
+// Test S3 filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package s3_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/s3"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*s3.FsObjectS3)(nil))
+	fstests.RemoteName = "TestS3:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

swift/swift.go

@@ -44,6 +44,9 @@ func init() {
 				Help:  "Memset Memstore UK v2",
 				Value: "https://auth.storage.memset.com/v2.0",
 			}},
+		}, {
+			Name: "tenant",
+			Help: "Tenant name - optional",
 		},
 		// snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
 		},
@@ -111,6 +114,7 @@ func swiftConnection(name string) (*swift.Connection, error) {
 		ApiKey:    apiKey,
 		AuthUrl:   authUrl,
 		UserAgent: fs.UserAgent,
+		Tenant:    fs.ConfigFile.MustValue(name, "tenant"),
 	}
 	err := c.Authenticate()
 	if err != nil {
@@ -157,7 +161,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
+func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
 	fs := &FsObjectSwift{
 		swift:  f,
 		remote: remote,
@@ -179,7 +183,7 @@ func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Obje
 //
 // May return nil if an error occurred
 func (f *FsSwift) NewFsObject(remote string) fs.Object {
-	return f.NewFsObjectWithInfo(remote, nil)
+	return f.newFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -201,8 +205,11 @@ func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
 			for i := range objects {
 				object := &objects[i]
 				// FIXME if there are no directories, swift gives back the files for some reason!
-				if directories && !strings.HasSuffix(object.Name, "/") {
-					continue
+				if directories {
+					if !strings.HasSuffix(object.Name, "/") {
+						continue
+					}
+					object.Name = object.Name[:len(object.Name)-1]
 				}
 				if !strings.HasPrefix(object.Name, f.root) {
 					fs.Log(f, "Odd name received %q", object.Name)
@@ -233,7 +240,7 @@ func (f *FsSwift) List() fs.ObjectsChan {
 		go func() {
 			defer close(out)
 			f.list(false, func(remote string, object *swift.Object) {
-				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
+				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})

swift/swift_test.go

@@ -0,0 +1,53 @@
+// Test Swift filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: go run gen_tests.go or make gen_tests
+package swift_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/swift"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*swift.FsObjectSwift)(nil))
+	fstests.RemoteName = "TestSwift:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }