Version 1.02

.gitignore

@@ -4,7 +4,6 @@ rclone
 rclonetest/rclonetest
 build
 docs/public
-MANUAL.md
-MANUAL.html
-MANUAL.txt
+README.html
+README.txt
 rclone.1

.travis.yml

@@ -1,10 +1,9 @@
 language: go
-sudo: false
 
 go:
+  - 1.1.2
   - 1.2.2
-  - 1.3.3
-  - 1.4
+  - 1.3
   - tip
 
 script:

Makefile

@@ -2,46 +2,39 @@ TAG := $(shell git describe --tags)
 LAST_TAG := $(shell git describe --tags --abbrev=0)
 NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 
-rclone:
+rclone:	*.go */*.go
 	@go version
-	go install -v ./...
+	go build
 
-test:	rclone
-	go test ./...
-	cd fs && ./test_all.sh
+doc:	rclone.1 README.html README.txt
 
-doc:	rclone.1 MANUAL.html MANUAL.txt
+rclone.1:	README.md
+	pandoc -s --from markdown --to man README.md -o rclone.1
 
-rclone.1:	MANUAL.md
-	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
+README.html:	README.md
+	pandoc -s --from markdown_github --to html README.md -o README.html
 
-MANUAL.md:	make_manual.py docs/content/*.md
-	./make_manual.py
-
-MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
-
-MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
+README.txt:	README.md
+	pandoc -s --from markdown_github --to plain README.md -o README.txt
 
 install: rclone
 	install -d ${DESTDIR}/usr/bin
-	install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone
+	install -t ${DESTDIR}/usr/bin rclone 
 
 clean:
 	go clean ./...
 	find . -name \*~ | xargs -r rm -f
 	rm -rf build docs/public
-	rm -f rclone rclonetest/rclonetest rclone.1 MANUAL.md MANUAL.html MANUAL.txt
+	rm -f rclone rclonetest/rclonetest rclone.1 README.html README.txt
 
 website:
 	cd docs && hugo
 
 upload_website:	website
-	rclone -v sync docs/public memstore:www-rclone-org
+	./rclone -v sync docs/public memstore:www-rclone-org
 
 upload:
-	rclone -v copy build/ memstore:downloads-rclone-org
+	./rclone -v copy build/ memstore:downloads-rclone-org
 
 cross:	doc
 	./cross-compile $(TAG)
@@ -55,15 +48,12 @@ tag:
 	echo -e "package fs\n const Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
 	perl -lpe 's/VERSION/${NEW_TAG}/g; s/DATE/'`date -I`'/g;' docs/content/downloads.md.in > docs/content/downloads.md
 	git tag $(NEW_TAG)
-	@echo "Add this to changelog in docs/content/changelog.md"
+	@echo "Add this to changelog in README.md"
 	@echo "  * $(NEW_TAG) -" `date -I`
 	@git log $(LAST_TAG)..$(NEW_TAG) --oneline
 	@echo "Then commit the changes"
-	@echo git commit -m \"Version $(NEW_TAG)\" -a -v
+	@echo git commit -m "Version $(NEW_TAG)" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
 	git tag -f $(LAST_TAG)
-
-gen_tests:
-	cd fstest/fstests && go run gen_tests.go

README.md

@@ -1,13 +1,12 @@
+% rclone(1) User Manual
+% Nick Craig-Wood
+% Jul 7, 2014
+
+Rclone
+======
+
 [![Logo](http://rclone.org/img/rclone-120x120.png)](http://rclone.org/)
 
-[Website](http://rclone.org) |
-[Documentation](http://rclone.org/docs/) |
-[Changelog](http://rclone.org/changelog/) |
-[Installation](http://rclone.org/install/) |
-[G+](https://google.com/+RcloneOrg)
-
-[![Build Status](https://travis-ci.org/ncw/rclone.png)](https://travis-ci.org/ncw/rclone) [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone)
-
 Rclone is a command line program to sync files and directories to and from
 
   * Google Drive
@@ -27,13 +26,300 @@ Features
   * Check mode to check all MD5SUMs
   * Can sync to and from network, eg two different Drive accounts
 
-See the home page for installation, usage, documentation, changelog
-and configuration walkthroughs.
+See the Home page for more documentation and configuration walkthroughs.
 
   * http://rclone.org/
 
+Install
+-------
+
+Rclone is a Go program and comes as a single binary file.
+
+Download the binary for your OS from
+
+  * http://rclone.org/downloads/
+
+Or alternatively if you have Go installed use
+
+    go install github.com/ncw/rclone
+
+and this will build the binary in `$GOPATH/bin`.
+
+Configure
+---------
+
+First you'll need to configure rclone.  As the object storage systems
+have quite complicated authentication these are kept in a config file
+`.rclone.conf` in your home directory by default.  (You can use the
+`--config` option to choose a different config file.)
+
+The easiest way to make the config is to run rclone with the config
+option, Eg
+
+    rclone config
+
+Usage
+-----
+
+Rclone syncs a directory tree from local to remote.
+
+Its basic syntax is
+
+    Syntax: [options] subcommand <parameters> <parameters...>
+
+See below for how to specify the source and destination paths.
+
+Subcommands
+-----------
+
+    rclone copy source:path dest:path
+
+Copy the source to the destination.  Doesn't transfer
+unchanged files, testing first by modification time then by
+MD5SUM.  Doesn't delete files from the destination.
+
+    rclone sync source:path dest:path
+
+Sync the source to the destination.  Doesn't transfer
+unchanged files, testing first by modification time then by
+MD5SUM.  Deletes any files that exist in source that don't
+exist in destination. Since this can cause data loss, test
+first with the `--dry-run` flag.
+
+    rclone ls [remote:path]
+
+List all the objects in the the path with sizes.
+
+    rclone lsl [remote:path]
+
+List all the objects in the the path with sizes and timestamps.
+
+    rclone lsd [remote:path]
+
+List all directories/objects/buckets in the the path.
+
+    rclone mkdir remote:path
+
+Make the path if it doesn't already exist
+
+    rclone rmdir remote:path
+
+Remove the path.  Note that you can't remove a path with
+objects in it, use purge for that.
+
+    rclone purge remote:path
+
+Remove the path and all of its contents.
+
+    rclone check source:path dest:path
+
+Checks the files in the source and destination match.  It
+compares sizes and MD5SUMs and prints a report of files which
+don't match.  It doesn't alter the source or destination.
+
+    rclone md5sum remote:path
+
+Produces an md5sum file for all the objects in the path.  This is in
+the same format as the standard md5sum tool produces.
+
+General options:
+
+```
+      --checkers=8: Number of checkers to run in parallel.
+      --config="~/.rclone.conf": Config file.
+  -n, --dry-run=false: Do a trial run with no permanent changes
+      --modify-window=1ns: Max time diff to be considered the same
+  -q, --quiet=false: Print as little stuff as possible
+      --stats=1m0s: Interval to print stats
+      --transfers=4: Number of file transfers to run in parallel.
+  -v, --verbose=false: Print lots more stuff
+```
+
+Developer options:
+
+```
+      --cpuprofile="": Write cpu profile to file
+```
+
+Local Filesystem
+----------------
+
+Paths are specified as normal filesystem paths, so
+
+    rclone sync /home/source /tmp/destination
+
+Will sync `/home/source` to `/tmp/destination`
+
+Swift / Rackspace cloudfiles / Memset Memstore
+----------------------------------------------
+
+Paths are specified as remote:container (or remote: for the `lsd`
+command.)  You may put subdirectories in too, eg
+`remote:container/path/to/dir`.
+
+So to copy a local directory to a swift container called backup:
+
+    rclone sync /home/source swift:backup
+
+The modified time is stored as metadata on the object as
+`X-Object-Meta-Mtime` as floating point since the epoch.
+
+This is a defacto standard (used in the official python-swiftclient
+amongst others) for storing the modification time (as read using
+os.Stat) for an object.
+
+Amazon S3
+---------
+
+Paths are specified as remote:bucket.  You may put subdirectories in
+too, eg `remote:bucket/path/to/dir`.
+
+So to copy a local directory to a s3 container called backup
+
+    rclone sync /home/source s3:backup
+
+The modified time is stored as metadata on the object as
+`X-Amz-Meta-Mtime` as floating point since the epoch.
+
+Google drive
+------------
+
+Paths are specified as remote:path  Drive paths may be as deep as required.
+
+The initial setup for drive involves getting a token from Google drive
+which you need to do in your browser.  `rclone config` walks you
+through it.
+
+To copy a local directory to a drive directory called backup
+
+    rclone copy /home/source remote:backup
+
+Google drive stores modification times accurate to 1 ms natively.
+
+Dropbox
+-------
+
+Paths are specified as remote:path  Dropbox paths may be as deep as required.
+
+The initial setup for dropbox involves getting a token from Dropbox
+which you need to do in your browser.  `rclone config` walks you
+through it.
+
+To copy a local directory to a drive directory called backup
+
+    rclone copy /home/source dropbox:backup
+
+Md5sums and timestamps in RFC3339 format accurate to 1ns are stored in
+a Dropbox datastore called "rclone".  Dropbox datastores are limited
+to 100,000 rows so this is the maximum number of files rclone can
+manage on Dropbox.
+
+Google Cloud Storage
+--------------------
+
+Paths are specified as remote:path Google Cloud Storage paths may be
+as deep as required.
+
+The initial setup for Google Cloud Storage involves getting a token
+from Google which you need to do in your browser.  `rclone config`
+walks you through it.
+
+To copy a local directory to a google cloud storage directory called backup
+
+    rclone copy /home/source remote:backup
+
+Google google cloud storage stores md5sums natively and rclone stores
+modification times as metadata on the object, under the "mtime" key in
+RFC3339 format accurate to 1ns.
+
+Single file copies
+------------------
+
+Rclone can copy single files
+
+    rclone src:path/to/file dst:path/dir
+
+Or
+
+    rclone src:path/to/file dst:path/to/file
+
+Note that you can't rename the file if you are copying from one file to another.
+
 License
 -------
 
 This is free software under the terms of MIT the license (check the
 COPYING file included in this package).
+
+Bugs
+----
+
+  * Drive: Sometimes get: Failed to copy: Upload failed: googleapi: Error 403: Rate Limit Exceeded
+    * quota is 100.0 requests/second/user
+  * Empty directories left behind with Local and Drive
+    * eg purging a local directory with subdirectories doesn't work
+
+Changelog
+---------
+  * v1.02 - 2014-07-19
+    * Implement Dropbox remote
+    * Implement Google Cloud Storage remote
+    * Verify Md5sums and Sizes after copies
+    * Remove times from "ls" command - lists sizes only
+    * Add add "lsl" - lists times and sizes
+    * Add "md5sum" command
+  * v1.01 - 2014-07-04
+    * drive: fix transfer of big files using up lots of memory
+  * v1.00 - 2014-07-03
+    * drive: fix whole second dates
+  * v0.99 - 2014-06-26
+    * Fix --dry-run not working
+    * Make compatible with go 1.1
+  * v0.98 - 2014-05-30
+    * s3: Treat missing Content-Length as 0 for some ceph installations
+    * rclonetest: add file with a space in
+  * v0.97 - 2014-05-05
+    * Implement copying of single files
+    * s3 & swift: support paths inside containers/buckets
+  * v0.96 - 2014-04-24
+    * drive: Fix multiple files of same name being created
+    * drive: Use o.Update and fs.Put to optimise transfers
+    * Add version number, -V and --version
+  * v0.95 - 2014-03-28
+    * rclone.org: website, docs and graphics
+    * drive: fix path parsing
+  * v0.94 - 2014-03-27
+    * Change remote format one last time
+    * GNU style flags
+  * v0.93 - 2014-03-16
+    * drive: store token in config file
+    * cross compile other versions
+    * set strict permissions on config file
+  * v0.92 - 2014-03-15
+    * Config fixes and --config option
+  * v0.91 - 2014-03-15
+    * Make config file
+  * v0.90 - 2013-06-27
+    * Project named rclone
+  * v0.00 - 2012-11-18
+    * Project started
+
+
+Contact and support
+-------------------
+
+The project website is at:
+
+  * https://github.com/ncw/rclone
+
+There you can file bug reports, ask for help or send pull requests.
+
+Authors
+-------
+
+  * Nick Craig-Wood <nick@craig-wood.com>
+
+Contributors
+------------
+
+  * Your name goes here!

RELEASE.md

@@ -1,19 +0,0 @@
-Required software for making a release
-  * [github-release](https://github.com/aktau/github-release) for uploading packages
-  * [gox](https://github.com/mitchellh/gox) for cross compiling
-    * Run `gox -build-toolchain`
-    * This assumes you have your own source checkout
-  * pandoc for making the html and man pages
-
-Making a release
-  * go get -u -f -v ./...
-  * make test
-  * make tag
-  * edit docs/content/changelog.md
-  * git commit -a -v
-  * make retag
-  * # Set the GOPATH for a gox enabled compiler - . ~/bin/go-cross
-  * make cross
-  * make upload
-  * make upload_website
-  * git push --tags origin master

cross-compile

@@ -21,8 +21,8 @@ mv build/rclone-${VERSION}-darwin-386 build/rclone-${VERSION}-osx-386
 cd build
 
 for d in `ls`; do
-    cp -a ../MANUAL.txt $d/README.txt
-    cp -a ../MANUAL.html $d/README.html
+    cp -a ../README.txt $d/
+    cp -a ../README.html $d/
     cp -a ../rclone.1 $d/
     zip -r9 $d.zip $d
     rm -rf $d

docs/content/authors.md

@@ -1,17 +0,0 @@
----
-title: "Authors"
-description: "Rclone Authors and Contributors"
-date: "2014-06-16"
----
-
-Authors
--------
-
-  * Nick Craig-Wood <nick@craig-wood.com>
-
-Contributors
-------------
-
-  * Alex Couper <amcouper@gmail.com>
-  * Leonid Shalupov <leonid@shalupov.com>
-  * Shimon Doodkin <helpmepro1@gmail.com>

docs/content/bugs.md

@@ -1,31 +0,0 @@
----
-title: "Bugs"
-description: "Rclone Bugs and Limitations"
-date: "2014-06-16"
----
-
-Bugs and Limitations
---------------------
-
-### Empty directories are left behind / not created ##
-
-With remotes that have a concept of directory, eg Local and Drive,
-empty directories may be left behind, or not created when one was
-expected.
-
-This is because rclone doesn't have a concept of a directory - it only
-works on objects.  Most of the object storage systems can't actually
-store a directory so there is nowhere for rclone to store anything
-about directories.
-
-You can work round this to some extent with the`purge` command which
-will delete everything under the path, **inluding** empty directories.
-
-This may be fixed at some point in
-[Issue #100](https://github.com/ncw/rclone/issues/100)
-
-### Directory timestamps aren't preserved ##
-
-For the same reason as the above, rclone doesn't have a concept of a
-directory - it only works on objects, therefore it can't preserve the
-timestamps of directories.

docs/content/changelog.md

@@ -1,139 +0,0 @@
----
-title: "Documentation"
-description: "Rclone Changelog"
-date: "2015-08-17"
----
-
-Changelog
----------
-
-  * v1.18 - 2015-08-17
-    * drive
-      * Add `--drive-use-trash` flag so rclone trashes instead of deletes
-      * Add "Forbidden to download" message for files with no downloadURL
-    * dropbox
-      * Remove datastore
-        * This was deprecated and it caused a lot of problems
-        * Modification times and MD5SUMs no longer stored
-      * Fix uploading files > 2GB
-    * s3
-      * use official AWS SDK from github.com/aws/aws-sdk-go
-      * **NB** will most likely require you to delete and recreate remote
-      * enable multipart upload which enables files > 5GB
-      * tested with Ceph / RadosGW / S3 emulation
-      * many thanks to Sam Liston and Brian Haymore at the [Utah
-        Center for High Performance Computing](https://www.chpc.utah.edu/) for a Ceph test account
-    * misc
-      * Show errors when reading the config file
-      * Do not print stats in quiet mode - thanks Leonid Shalupov
-      * Add FAQ
-      * Fix created directories not obeying umask
-      * Linux installation instructions - thanks Shimon Doodkin
-  * v1.17 - 2015-06-14
-    * dropbox: fix case insensitivity issues - thanks Leonid Shalupov
-  * v1.16 - 2015-06-09
-    * Fix uploading big files which was causing timeouts or panics
-    * Don't check md5sum after download with --size-only
-  * v1.15 - 2015-06-06
-    * Add --checksum flag to only discard transfers by MD5SUM - thanks Alex Couper
-    * Implement --size-only flag to sync on size not checksum & modtime
-    * Expand docs and remove duplicated information
-    * Document rclone's limitations with directories
-    * dropbox: update docs about case insensitivity
-  * v1.14 - 2015-05-21
-    * local: fix encoding of non utf-8 file names - fixes a duplicate file problem
-    * drive: docs about rate limiting
-    * google cloud storage: Fix compile after API change in "google.golang.org/api/storage/v1"
-  * v1.13 - 2015-05-10
-    * Revise documentation (especially sync)
-    * Implement --timeout and --conntimeout
-    * s3: ignore etags from multipart uploads which aren't md5sums
-  * v1.12 - 2015-03-15
-    * drive: Use chunked upload for files above a certain size
-    * drive: add --drive-chunk-size and --drive-upload-cutoff parameters
-    * drive: switch to insert from update when a failed copy deletes the upload
-    * core: Log duplicate files if they are detected
-  * v1.11 - 2015-03-04
-    * swift: add region parameter
-    * drive: fix crash on failed to update remote mtime
-    * In remote paths, change native directory separators to /
-    * Add synchronization to ls/lsl/lsd output to stop corruptions
-    * Ensure all stats/log messages to go stderr
-    * Add --log-file flag to log everything (including panics) to file
-    * Make it possible to disable stats printing with --stats=0
-    * Implement --bwlimit to limit data transfer bandwidth
-  * v1.10 - 2015-02-12
-    * s3: list an unlimited number of items
-    * Fix getting stuck in the configurator
-  * v1.09 - 2015-02-07
-    * windows: Stop drive letters (eg C:) getting mixed up with remotes (eg drive:)
-    * local: Fix directory separators on Windows
-    * drive: fix rate limit exceeded errors
-  * v1.08 - 2015-02-04
-    * drive: fix subdirectory listing to not list entire drive
-    * drive: Fix SetModTime
-    * dropbox: adapt code to recent library changes
-  * v1.07 - 2014-12-23
-    * google cloud storage: fix memory leak
-  * v1.06 - 2014-12-12
-    * Fix "Couldn't find home directory" on OSX
-    * swift: Add tenant parameter
-    * Use new location of Google API packages
-  * v1.05 - 2014-08-09
-    * Improved tests and consequently lots of minor fixes
-    * core: Fix race detected by go race detector
-    * core: Fixes after running errcheck
-    * drive: reset root directory on Rmdir and Purge
-    * fs: Document that Purger returns error on empty directory, test and fix
-    * google cloud storage: fix ListDir on subdirectory
-    * google cloud storage: re-read metadata in SetModTime
-    * s3: make reading metadata more reliable to work around eventual consistency problems
-    * s3: strip trailing / from ListDir()
-    * swift: return directories without / in ListDir
-  * v1.04 - 2014-07-21
-    * google cloud storage: Fix crash on Update
-  * v1.03 - 2014-07-20
-    * swift, s3, dropbox: fix updated files being marked as corrupted
-    * Make compile with go 1.1 again
-  * v1.02 - 2014-07-19
-    * Implement Dropbox remote
-    * Implement Google Cloud Storage remote
-    * Verify Md5sums and Sizes after copies
-    * Remove times from "ls" command - lists sizes only
-    * Add add "lsl" - lists times and sizes
-    * Add "md5sum" command
-  * v1.01 - 2014-07-04
-    * drive: fix transfer of big files using up lots of memory
-  * v1.00 - 2014-07-03
-    * drive: fix whole second dates
-  * v0.99 - 2014-06-26
-    * Fix --dry-run not working
-    * Make compatible with go 1.1
-  * v0.98 - 2014-05-30
-    * s3: Treat missing Content-Length as 0 for some ceph installations
-    * rclonetest: add file with a space in
-  * v0.97 - 2014-05-05
-    * Implement copying of single files
-    * s3 & swift: support paths inside containers/buckets
-  * v0.96 - 2014-04-24
-    * drive: Fix multiple files of same name being created
-    * drive: Use o.Update and fs.Put to optimise transfers
-    * Add version number, -V and --version
-  * v0.95 - 2014-03-28
-    * rclone.org: website, docs and graphics
-    * drive: fix path parsing
-  * v0.94 - 2014-03-27
-    * Change remote format one last time
-    * GNU style flags
-  * v0.93 - 2014-03-16
-    * drive: store token in config file
-    * cross compile other versions
-    * set strict permissions on config file
-  * v0.92 - 2014-03-15
-    * Config fixes and --config option
-  * v0.91 - 2014-03-15
-    * Make config file
-  * v0.90 - 2013-06-27
-    * Project named rclone
-  * v0.00 - 2012-11-18
-    * Project started

docs/content/contact.md

@@ -5,17 +5,8 @@ date: "2014-04-26"
 ---
 
 Contact the rclone project
---------------------------
-
-The project website is at:
-
-  * https://github.com/ncw/rclone
-
-There you can file bug reports, ask for help or contribute pull
-requests.
-
-See also
 
+  * [Github project page for source, reporting bugs and pull requests](http://github.com/ncw/rclone)
   * <a href="https://google.com/+RcloneOrg" rel="publisher">Google+ page for general comments</a></li>
 
 Or email [Nick Craig-Wood](mailto:nick@craig-wood.com)

docs/content/docs.md

@@ -1,9 +1,22 @@
 ---
 title: "Documentation"
-description: "Rclone Usage"
-date: "2015-06-06"
+description: "Rclone Documentation"
+date: "2014-07-17"
 ---
 
+Install
+-------
+
+Rclone is a Go program and comes as a single binary file.
+
+[Download](/downloads/) the relevant binary.
+
+Or alternatively if you have Go installed use
+
+    go get github.com/ncw/rclone
+
+and this will build the binary in `$GOPATH/bin`.
+
 Configure
 ---------
 
@@ -17,13 +30,11 @@ option:
 
     rclone config
 
-See the following for detailed instructions for
+See below for detailed instructions for
 
   * [Google drive](/drive/)
   * [Amazon S3](/s3/)
   * [Swift / Rackspace Cloudfiles / Memset Memstore](/swift/)
-  * [Dropbox](/dropbox/)
-  * [Google Cloud Storage](/googlcloudstorage/)
   * [Local filesystem](/local/)
 
 Usage
@@ -44,216 +55,104 @@ You can define as many storage paths as you like in the config file.
 Subcommands
 -----------
 
-### rclone copy source:path dest:path ###
+    rclone copy source:path dest:path
 
 Copy the source to the destination.  Doesn't transfer
-unchanged files, testing by size and modification time or
+unchanged files, testing first by modification time then by
 MD5SUM.  Doesn't delete files from the destination.
 
-### rclone sync source:path dest:path ###
+    rclone sync source:path dest:path
 
-Sync the source to the destination, changing the destination
-only.  Doesn't transfer unchanged files, testing by size and
-modification time or MD5SUM.  Destination is updated to match
-source, including deleting files if necessary.  Since this can
-cause data loss, test first with the `--dry-run` flag.
+Sync the source to the destination.  Doesn't transfer
+unchanged files, testing first by modification time then by
+MD5SUM.  Deletes any files that exist in source that don't
+exist in destination. Since this can cause data loss, test
+first with the -dry-run flag.
 
-### rclone ls [remote:path] ###
+    rclone ls [remote:path]
 
-List all the objects in the the path with size and path.
+List all the objects in the the path with sizes.
 
-### rclone lsd [remote:path] ###
+    rclone lsl [remote:path]
 
-List all directories/containers/buckets in the the path.
+List all the objects in the the path with sizes and timestamps.
 
-### rclone lsl [remote:path] ###
+    rclone lsd [remote:path]
 
-List all the objects in the the path with modification time,
-size and path.
+List all directories/objects/buckets in the the path.
 
-### rclone md5sum [remote:path] ###
-
-Produces an md5sum file for all the objects in the path.  This
-is in the same format as the standard md5sum tool produces.
-
-### rclone mkdir remote:path ###
+    rclone mkdir remote:path
 
 Make the path if it doesn't already exist
 
-### rclone rmdir remote:path ###
+    rclone rmdir remote:path
 
 Remove the path.  Note that you can't remove a path with
 objects in it, use purge for that.
 
-### rclone purge remote:path ###
+    rclone purge remote:path
 
 Remove the path and all of its contents.
 
-### rclone check source:path dest:path ###
+    rclone check source:path dest:path
 
 Checks the files in the source and destination match.  It
 compares sizes and MD5SUMs and prints a report of files which
 don't match.  It doesn't alter the source or destination.
 
-### rclone config ###
+    rclone md5sum remote:path
 
-Enter an interactive configuration session.
+Produces an md5sum file for all the objects in the path.  This is in
+the same format as the standard md5sum tool produces.
+General options:
 
-### rclone help ###
+```
+      --checkers=8: Number of checkers to run in parallel.
+      --transfers=4: Number of file transfers to run in parallel.
+      --config="~/.rclone.conf": Config file.
+  -n, --dry-run=false: Do a trial run with no permanent changes
+      --modify-window=1ns: Max time diff to be considered the same
+  -q, --quiet=false: Print as little stuff as possible
+      --stats=1m0s: Interval to print stats
+  -v, --verbose=false: Print lots more stuff
+```
 
-Prints help on rclone commands and options.
+Developer options:
 
-Options
+```
+      --cpuprofile="": Write cpu profile to file
+```
+
+License
 -------
 
-Rclone has a number of options to control its behaviour.
+This is free software under the terms of MIT the license (check the
+COPYING file included in this package).
 
-Options which use TIME use the go time parser.  A duration string is a
-possibly signed sequence of decimal numbers, each with optional
-fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid
-time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+Bugs
+----
 
-Options which use SIZE use kByte by default.  However a suffix of `k`
-for kBytes, `M` for MBytes and `G` for GBytes may be used.  These are
-the binary units, eg 2**10, 2**20, 2**30 respectively.
+  * Doesn't sync individual files yet, only directories.
+  * Drive: Sometimes get: Failed to copy: Upload failed: googleapi: Error 403: Rate Limit Exceeded
+    * quota is 100.0 requests/second/user
+  * Empty directories left behind with Local and Drive
+    * eg purging a local directory with subdirectories doesn't work
 
-### --bwlimit=SIZE ###
+Contact and support
+-------------------
 
-Bandwidth limit in kBytes/s, or use suffix k|M|G.  The default is `0`
-which means to not limit bandwidth.
+The project website is at:
 
-For example to limit bandwidth usage to 10 MBytes/s use `--bwlimit 10M`
+  * https://github.com/ncw/rclone
 
-This only limits the bandwidth of the data transfer, it doesn't limit
-the bandwith of the directory listings etc.
+There you can file bug reports, ask for help or contribute patches.
 
-### --checkers=N ###
+Authors
+-------
 
-The number of checkers to run in parallel.  Checkers do the equality
-checking of files during a sync.  For some storage systems (eg s3,
-swift, dropbox) this can take a significant amount of time so they are
-run in parallel.
+  * Nick Craig-Wood <nick@craig-wood.com>
 
-The default is to run 8 checkers in parallel.
+Contributors
+------------
 
-### -c, --checksum ###
-
-Normally rclone will look at modification time and size of files to
-see if they are equal.  If you set this flag then rclone will check
-MD5SUM and size to determine if files are equal.
-
-This is very useful when transferring between remotes which store the
-MD5SUM on the object which include swift, s3, drive, and google cloud
-storage.
-
-Eg `rclone --checksum sync s3:/bucket swift:/bucket` would run much
-quicker than without the `--checksum` flag.
-
-When using this flag, rclone won't update mtimes of remote files if
-they are incorrect as it would normally.
-
-### --config=CONFIG_FILE ###
-
-Specify the location of the rclone config file.  Normally this is in
-your home directory as a file called `.rclone.conf`.  If you run
-`rclone -h` and look at the help for the `--config` option you will
-see where the default location is for you.  Use this flag to override
-the config location, eg `rclone --config=".myconfig" .config`.
-
-### --contimeout=TIME ###
-
-Set the connection timeout. This should be in go time format which
-looks like `5s` for 5 seconds, `10m` for 10 minutes, or `3h30m`.
-
-The connection timeout is the amount of time rclone will wait for a
-connection to go through to a remote object storage system.  It is
-`1m` by default.
-
-### -n, --dry-run ###
-
-Do a trial run with no permanent changes.  Use this in combination
-with the `-v` flag to see what rclone would do without actually doing
-it.  Useful when setting up the `sync` command.
-
-### --log-file=FILE ###
-
-Log all of rclone's output to FILE.  This is not active by default.
-This can be useful for tracking down problems with syncs in
-combination with the `-v` flag.
-
-### --modify-window=TIME ###
-
-When checking whether a file has been modified, this is the maximum
-allowed time difference that a file can have and still be considered
-equivalent.
-
-The default is `1ns` unless this is overridden by a remote.  For
-example OS X only stores modification times to the nearest second so
-if you are reading and writing to an OS X filing system this will be
-`1s` by default.
-
-This command line flag allows you to override that computed default.
-
-### -q, --quiet ###
-
-Normally rclone outputs stats and a completion message.  If you set
-this flag it will make as little output as possible.
-
-### --size-only ###
-
-Normally rclone will look at modification time and size of files to
-see if they are equal.  If you set this flag then rclone will check
-only the size.
-
-This can be useful transferring files from dropbox which have been
-modified by the desktop sync client which doesn't set checksums of
-modification times in the same way as rclone.
-
-When using this flag, rclone won't update mtimes of remote files if
-they are incorrect as it would normally.
-
-### --stats=TIME ###
-
-Rclone will print stats at regular intervals to show its progress.
-
-This sets the interval.
-
-The default is `1m`. Use 0 to disable.
-
-### --timeout=TIME ###
-
-This sets the IO idle timeout.  If a transfer has started but then
-becomes idle for this long it is considered broken and disconnected.
-
-The default is `5m`.  Set to 0 to disable.
-
-### --transfers=N ###
-
-The number of file transfers to run in parallel.  It can sometimes be
-useful to set this to a smaller number if the remote is giving a lot
-of timeouts or bigger if you have lots of bandwidth and a fast remote.
-
-The default is to run 4 file transfers in parallel.
-
-### -v, --verbose ###
-
-If you set this flag, rclone will become very verbose telling you
-about every file it considers and transfers.
-
-Very useful for debugging.
-
-### -V, --version ###
-
-Prints the version number
-
-Developer options
------------------
-
-These options are useful when developing or debugging rclone.  There
-are also some more remote specific options which aren't documented
-here which are used for testing.  These start with remote name eg
-`--drive-test-option`.
-
-### --cpuprofile=FILE ###
-
-Write cpu profile to file.  This can be analysed with `go tool pprof`.
+  * Your name goes here!

docs/content/downloads.md

@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2015-08-17"
+date: "2014-07-19"
 ---
 
-Rclone Download v1.18
+Rclone Download v1.02
 =====================
 
   * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-windows-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-windows-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-windows-amd64.zip)
   * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-osx-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-osx-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-osx-amd64.zip)
   * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.18-linux-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-linux-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-linux-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.02-linux-arm.zip)
   * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.18-freebsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-freebsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-freebsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.02-freebsd-arm.zip)
   * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.18-netbsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-netbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-netbsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.02-netbsd-arm.zip)
   * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.18-openbsd-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-openbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.02-openbsd-amd64.zip)
   * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.18-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.02-plan9-386.zip)
 
 Older downloads can be found [here](http://downloads.rclone.org/)

docs/content/drive.md

@@ -1,7 +1,7 @@
 ---
 title: "Google drive"
 description: "Rclone docs for Google drive"
-date: "2015-05-10"
+date: "2014-04-26"
 ---
 
 <i class="fa fa-google"></i> Google Drive
@@ -69,31 +69,7 @@ To copy a local directory to a drive directory called backup
 
     rclone copy /home/source remote:backup
 
-### Modified time ###
+Modified time
+-------------
 
 Google drive stores modification times accurate to 1 ms.
-
-### Revisions ###
-
-Google drive stores revisions of files.  When you upload a change to
-an existing file to google drive using rclone it will create a new
-revision of that file.
-
-Revisions follow the standard google policy which at time of writing
-was
-
-  * They are deleted after 30 days or 100 revisions (whatever comes first).
-  * They do not count towards a user storage quota.
-
-### Deleting files ###
-
-By default rclone will delete files permanently when requested.  If
-sending them to the trash is required instead then use the
-`--drive-use-trash` flag.
-
-### Limitations ###
-
-Drive has quite a lot of rate limiting.  This causes rclone to be
-limited to transferring about 2 files per second only.  Individual
-files may be transferred much faster at 100s of MBytes/s but lots of
-small files can take a long time.

docs/content/dropbox.md

@@ -71,12 +71,10 @@ To copy a local directory to a dropbox directory called backup
 
     rclone copy /home/source remote:backup
 
-### Modified time and MD5SUMs ###
+Modified time
+-------------
 
-Dropbox doesn't have the capability of storing modification times or
-MD5SUMs so syncs will effectively have the `--size-only` flag set.
-
-### Limitations ###
-
-Note that Dropbox is case sensitive so you can't have a file called
-"Hello.doc" and one called "hello.doc".
+Md5sums and timestamps in RFC3339 format accurate to 1ns are stored in
+a Dropbox datastore called "rclone".  Dropbox datastores are limited
+to 100,000 rows so this is the maximum number of files rclone can
+manage on Dropbox.

docs/content/faq.md

@@ -1,50 +0,0 @@
----
-title: "FAQ"
-description: "Rclone Frequently Asked Questions"
-date: "2015-06-06"
----
-
-Frequently Asked Questions
---------------------------
-
-### Do all cloud storage systems support all rclone commands ###
-
-Yes they do.  All the rclone commands (eg `sync`, `copy` etc) will
-work on all the remote storage systems.
-
-
-### Can rclone sync directly from drive to s3 ###
-
-Rclone can sync between two remote cloud storage systems just fine.
-
-Note that it effectively downloads the file and uploads it again, so
-the node running rclone would need to have lots of bandwidth.
-
-The syncs would be incremental (on a file by file basis).
-
-Eg
-
-    rclone sync drive:Folder s3:bucket
-
-
-### Using rclone from multiple locations at the same time ###
-
-You can use rclone from multiple places at the same time if you choose
-different subdirectory for the output, eg
-
-```
-Server A> rclone sync /tmp/whatever remote:ServerA
-Server B> rclone sync /tmp/whatever remote:ServerB
-```
-
-If you sync to the same directory then you should use rclone copy
-otherwise the two rclones may delete each others files, eg
-
-```
-Server A> rclone copy /tmp/whatever remote:Backup
-Server B> rclone copy /tmp/whatever remote:Backup
-```
-
-The file names you upload from Server A and Server B should be
-different in this case, otherwise some file systems (eg Drive) may
-make duplicates.

docs/content/googlecloudstorage.md

@@ -109,7 +109,8 @@ files in the bucket.
 
     rclone sync /home/local/directory remote:bucket
 
-### Modified time ###
+Modified time
+-------------
 
 Google google cloud storage stores md5sums natively and rclone stores
 modification times as metadata on the object, under the "mtime" key in

docs/content/install.md

@@ -1,39 +0,0 @@
----
-title: "Install"
-description: "Rclone Installation"
-date: "2015-06-12"
----
-
-Install
--------
-
-Rclone is a Go program and comes as a single binary file.
-
-[Download](/downloads/) the relevant binary.
-
-Or alternatively if you have Go installed use
-
-    go get github.com/ncw/rclone
-
-and this will build the binary in `$GOPATH/bin`.  If you have built
-rclone before then you will want to update its dependencies first with
-this (remove `-f` if using go < 1.4)
-
-    go get -u -v -f github.com/ncw/rclone/...
-
-See the [Usage section](/docs/) of the docs for how to use rclone, or
-run `rclone -h`.
-
-linux binary downloaded files install example
--------
-
-    unzip rclone-v1.17-linux-amd64.zip
-    cd rclone-v1.17-linux-amd64
-    #copy binary file
-    sudo cp rclone /usr/sbin/
-    sudo chown root:root /usr/sbin/rclone
-    sudo chmod 755 /usr/sbin/rclone
-    #install manpage
-    sudo mkdir -p /usr/local/share/man/man1
-    sudo cp rclone.1 /usr/local/share/man/man1/
-    sudo mandb 

docs/content/licence.md

@@ -1,34 +0,0 @@
----
-title: "Licence"
-description: "Rclone Licence"
-date: "2015-06-06"
----
-
-License
--------
-
-This is free software under the terms of MIT the license (check the
-COPYING file included with the source code).
-
-```
-Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-```
-

docs/content/local.md

@@ -16,23 +16,10 @@ Will sync `/home/source` to `/tmp/destination`
 These can be configured into the config file for consistencies sake,
 but it is probably easier not to.
 
-### Modified time ###
+Modified time
+-------------
 
 Rclone reads and writes the modified time using an accuracy determined by
 the OS.  Typically this is 1ns on Linux, 10 ns on Windows and 1 Second
 on OS X.
 
-### Filenames ###
-
-Filenames are expected to be encoded in UTF-8 on disk.  This is the
-normal case for Windows and OS X.  There is a bit more uncertainty in
-the Linux world, but new distributions will have UTF-8 encoded files
-names.
-
-If an invalid (non-UTF8) filename is read, the invalid caracters will
-be replaced with the unicode replacement character, '<27>'.  `rclone`
-will emit a debug message in this case (use `-v` to see), eg
-
-```
-Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf"
-```

docs/content/s3.md

@@ -27,55 +27,41 @@ Choose a number from below
  1) swift
  2) s3
  3) local
- 4) google cloud storage
- 5) dropbox
- 6) drive
+ 4) drive
 type> 2
 AWS Access Key ID.
 access_key_id> accesskey
 AWS Secret Access Key (password). 
 secret_access_key> secretaccesskey
-Region to connect to.
+Endpoint for S3 API.
 Choose a number from below, or type in your own value
  * The default endpoint - a good choice if you are unsure.
  * US Region, Northern Virginia or Pacific Northwest.
  * Leave location constraint empty.
- 1) us-east-1
- * US West (Oregon) Region
- * Needs location constraint us-west-2.
- 2) us-west-2
+ 1) https://s3.amazonaws.com/
+ * US Region, Northern Virginia only.
+ * Leave location constraint empty.
+ 2) https://s3-external-1.amazonaws.com
 [snip]
  * South America (Sao Paulo) Region
  * Needs location constraint sa-east-1.
- 9) sa-east-1
- * If using an S3 clone that only understands v2 signatures - eg Ceph - set this and make sure you set the endpoint.
-10) other-v2-signature
- * If using an S3 clone that understands v4 signatures set this and make sure you set the endpoint.
-11) other-v4-signature
-region> 1
-Endpoint for S3 API.
-Leave blank if using AWS to use the default endpoint for the region.
-Specify if using an S3 clone such as Ceph.
-endpoint> 
-Location constraint - must be set to match the Region. Used when creating buckets only.
+ 9) https://s3-sa-east-1.amazonaws.com
+endpoint> 1
+Location constraint - must be set to match the Endpoint.
 Choose a number from below, or type in your own value
  * Empty for US Region, Northern Virginia or Pacific Northwest.
  1) 
  * US West (Oregon) Region.
  2) us-west-2
- * US West (Northern California) Region.
- 3) us-west-1
- * EU (Ireland) Region.
- 4) eu-west-1
 [snip]
+ * South America (Sao Paulo) Region.
+ 9) sa-east-1
 location_constraint> 1
-Remote config
 --------------------
 [remote]
 access_key_id = accesskey
 secret_access_key = secretaccesskey
-region = us-east-1
-endpoint = 
+endpoint = https://s3.amazonaws.com/
 location_constraint = 
 --------------------
 y) Yes this is OK
@@ -114,52 +100,8 @@ files in the bucket.
 
     rclone sync /home/local/directory remote:bucket
 
-### Modified time ###
+Modified time
+-------------
 
 The modified time is stored as metadata on the object as
 `X-Amz-Meta-Mtime` as floating point since the epoch accurate to 1 ns.
-
-### Multipart uploads ###
-
-rclone supports multipart uploads with S3 which means that it can
-upload files bigger than 5GB. Note that files uploaded with multipart
-upload don't have an MD5SUM.
-
-### Ceph ###
-
-Ceph is an object storage system which presents an Amazon S3 interface.
-
-To use rclone with ceph, you need to set the following parameters in
-the config.
-
-```
-access_key_id = Whatever
-secret_access_key = Whatever
-endpoint = https://ceph.endpoint.goes.here/
-region = other-v2-signature
-```
-
-Note also that Ceph sometimes puts `/` in the passwords it gives
-users.  If you read the secret access key using the command line tools
-you will get a JSON blob with the `/` escaped as `\/`.  Make sure you
-only write `/` in the secret access key.
-
-Eg the dump from Ceph looks something like this (irrelevant keys
-removed).
-
-```
-{
-    "user_id": "xxx",
-    "display_name": "xxxx",
-    "keys": [
-        {
-            "user": "xxx",
-            "access_key": "xxxxxx",
-            "secret_key": "xxxxxx\/xxxx"
-        }
-    ],
-}
-```
-
-Because this is a json dump, it is encoding the `/` as `\/`, so if you
-use the secret key as `xxxxxx/xxxx`  it will work fine.

docs/content/swift.md

@@ -52,15 +52,12 @@ Choose a number from below, or type in your own value
  * Memset Memstore UK v2
  5) https://auth.storage.memset.com/v2.0
 auth> 1
-Tenant name - optional
-tenant>
 Remote config
 --------------------
 [remote]
 user = user_name
 key = password_or_api_key
 auth = https://auth.api.rackspacecloud.com/v1.0
-tenant =
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -87,7 +84,8 @@ excess files in the container.
 
     rclone sync /home/local/directory remote:container
 
-### Modified time ###
+Modified time
+-------------
 
 The modified time is stored as metadata on the object as
 `X-Object-Meta-Mtime` as floating point since the epoch accurate to 1

docs/layouts/chrome/meta.html

@@ -3,7 +3,6 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="{{ .Description }}">
     <meta name="author" content="Nick Craig-Wood">
-    <link rel="shortcut icon" type="image/png" href="/img/rclone-16x16.png"/>
     <script>
       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),

docs/layouts/chrome/navbar.html

@@ -12,18 +12,8 @@
             <div class="collapse navbar-collapse navbar-ex1-collapse">
               <ul class="nav navbar-nav">
                 <li><a href="/downloads/"><i class="fa fa-cloud-download"></i> Downloads</a></li>
-                <li class="dropdown">
-                  <a href="#" class="dropdown-toggle" data-toggle="dropdown"><b class="caret"></b> Docs</a>
-                  <ul class="dropdown-menu">
-                    <li><a href="/install/"><i class="fa fa-book"></i> Installation</a></li>
-                    <li><a href="/docs/"><i class="fa fa-book"></i> Usage</a></li>
-                    <li><a href="/changelog/"><i class="fa fa-book"></i> Changelog</a></li>
-                    <li><a href="/bugs/"><i class="fa fa-book"></i> Bugs</a></li>
-                    <li><a href="/faq/"><i class="fa fa-book"></i> FAQ</a></li>
-                    <li><a href="/licence/"><i class="fa fa-book"></i> Licence</a></li>
-                    <li><a href="/authors/"><i class="fa fa-book"></i> Authors</a></li>
-                  </ul>
-                </li>
+                <li><a href="/docs/"><i class="fa fa-book"></i> Docs</a></li>
+                <li><a href="/contact/"><i class="fa fa-envelope"></i> Contact</a></li>
                 <li class="dropdown">
                   <a href="#" class="dropdown-toggle" data-toggle="dropdown"><b class="caret"></b> Storage Systems</a>
                   <ul class="dropdown-menu">
@@ -35,7 +25,6 @@
                     <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
                   </ul>
                 </li>
-                <li><a href="/contact/"><i class="fa fa-envelope"></i> Contact</a></li>
               </ul>
             </div>
         </div>

drive/drive.go

@@ -1,6 +1,14 @@
 // Drive interface
 package drive
 
+// Gets this quite often
+// Failed to set mtime: googleapi: Error 403: Rate Limit Exceeded
+
+// FIXME list containers equivalent should list directories?
+
+// FIXME list directory should list to channel for concurrency not
+// append to array
+
 // FIXME need to deal with some corner cases
 // * multiple files with the same name
 // * files can be in multiple directories
@@ -10,14 +18,16 @@ package drive
 import (
 	"fmt"
 	"io"
+	"log"
+	"mime"
 	"net/http"
+	"os"
+	"path"
 	"strings"
 	"sync"
 	"time"
 
-	"google.golang.org/api/drive/v2"
-	"google.golang.org/api/googleapi"
-
+	"code.google.com/p/google-api-go-client/drive/v2"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/googleauth"
 	"github.com/ogier/pflag"
@@ -28,22 +38,14 @@ const (
 	rcloneClientId     = "202264815644.apps.googleusercontent.com"
 	rcloneClientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
 	driveFolderType    = "application/vnd.google-apps.folder"
-	timeFormatIn       = time.RFC3339
-	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
-	minSleep           = 10 * time.Millisecond
-	maxSleep           = 2 * time.Second
-	decayConstant      = 2 // bigger for slower decay, exponential
+	RFC3339In          = time.RFC3339
+	RFC3339Out         = "2006-01-02T15:04:05.000000000Z07:00"
 )
 
 // Globals
 var (
 	// Flags
 	driveFullList = pflag.BoolP("drive-full-list", "", true, "Use a full listing for directory list. More data but usually quicker.")
-	driveUseTrash = pflag.BoolP("drive-use-trash", "", false, "Send files to the trash instead of deleting permanently.")
-	// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
-	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
-	chunkSize         = fs.SizeSuffix(256 * 1024)
-	driveUploadCutoff = chunkSize
 	// Description of how to auth for this app
 	driveAuth = &googleauth.Auth{
 		Scope:               "https://www.googleapis.com/auth/drive",
@@ -68,8 +70,6 @@ func init() {
 			Help: "Google Application Client Secret - leave blank to use rclone's.",
 		}},
 	})
-	pflag.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
-	pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
 }
 
 // FsDrive represents a remote drive server
@@ -81,10 +81,8 @@ type FsDrive struct {
 	rootId       string         // Id of the root directory
 	foundRoot    bool           // Whether we have found the root or not
 	findRootLock sync.Mutex     // Protect findRoot from concurrent use
-	dirCache     *dirCache      // Map of directory path to directory id
+	dirCache     dirCache       // Map of directory path to directory id
 	findDirLock  sync.Mutex     // Protect findDir from concurrent use
-	pacer        chan struct{}  // To pace the operations
-	sleepTime    time.Duration  // Time to sleep for each transaction
 }
 
 // FsObjectDrive describes a drive object
@@ -106,8 +104,8 @@ type dirCache struct {
 }
 
 // Make a new locked map
-func newDirCache() *dirCache {
-	d := &dirCache{}
+func newDirCache() dirCache {
+	d := dirCache{}
 	d.Flush()
 	return d
 }
@@ -151,97 +149,6 @@ func (f *FsDrive) String() string {
 	return fmt.Sprintf("Google drive root '%s'", f.root)
 }
 
-// Start a call to the drive API
-//
-// This must be called as a pair with endCall
-//
-// This waits for the pacer token
-func (f *FsDrive) beginCall() {
-	// pacer starts with a token in and whenever we take one out
-	// XXX ms later we put another in.  We could do this with a
-	// Ticker more accurately, but then we'd have to work out how
-	// not to run it when it wasn't needed
-	<-f.pacer
-
-	// Restart the timer
-	go func(t time.Duration) {
-		// fs.Debug(f, "New sleep for %v at %v", t, time.Now())
-		time.Sleep(t)
-		f.pacer <- struct{}{}
-	}(f.sleepTime)
-}
-
-// End a call to the drive API
-//
-// Refresh the pace given an error that was returned.  It returns a
-// boolean as to whether the operation should be retried.
-//
-// See https://developers.google.com/drive/web/handle-errors
-// http://stackoverflow.com/questions/18529524/403-rate-limit-after-only-1-insert-per-second
-func (f *FsDrive) endCall(err error) bool {
-	again := false
-	oldSleepTime := f.sleepTime
-	if err == nil {
-		f.sleepTime = (f.sleepTime<<decayConstant - f.sleepTime) >> decayConstant
-		if f.sleepTime < minSleep {
-			f.sleepTime = minSleep
-		}
-		if f.sleepTime != oldSleepTime {
-			fs.Debug(f, "Reducing sleep to %v", f.sleepTime)
-		}
-	} else {
-		fs.Debug(f, "Error recived: %T %#v", err, err)
-		// Check for net error Timeout()
-		if x, ok := err.(interface {
-			Timeout() bool
-		}); ok && x.Timeout() {
-			again = true
-		}
-		// Check for net error Temporary()
-		if x, ok := err.(interface {
-			Temporary() bool
-		}); ok && x.Temporary() {
-			again = true
-		}
-		switch gerr := err.(type) {
-		case *googleapi.Error:
-			if gerr.Code >= 500 && gerr.Code < 600 {
-				// All 5xx errors should be retried
-				again = true
-			} else if len(gerr.Errors) > 0 {
-				reason := gerr.Errors[0].Reason
-				if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
-					again = true
-				}
-			}
-		}
-	}
-	if again {
-		f.sleepTime *= 2
-		if f.sleepTime > maxSleep {
-			f.sleepTime = maxSleep
-		}
-		if f.sleepTime != oldSleepTime {
-			fs.Debug(f, "Rate limited, increasing sleep to %v", f.sleepTime)
-		}
-	}
-	return again
-}
-
-// Pace the remote operations to not exceed Google's limits and retry
-// on 403 rate limit exceeded
-//
-// This calls fn, expecting it to place its error in perr
-func (f *FsDrive) call(perr *error, fn func()) {
-	for {
-		f.beginCall()
-		fn()
-		if !f.endCall(*perr) {
-			break
-		}
-	}
-}
-
 // parseParse parses a drive 'url'
 func parseDrivePath(path string) (root string, err error) {
 	root = strings.Trim(path, "/")
@@ -279,10 +186,7 @@ func (f *FsDrive) listAll(dirId string, title string, directoriesOnly bool, file
 	list := f.svc.Files.List().Q(query).MaxResults(1000)
 OUTER:
 	for {
-		var files *drive.FileList
-		f.call(&err, func() {
-			files, err = list.Do()
-		})
+		files, err := list.Do()
 		if err != nil {
 			return false, fmt.Errorf("Couldn't list directory: %s", err)
 		}
@@ -300,27 +204,8 @@ OUTER:
 	return
 }
 
-// Returns true of x is a power of 2 or zero
-func isPowerOfTwo(x int64) bool {
-	switch {
-	case x == 0:
-		return true
-	case x < 0:
-		return false
-	default:
-		return (x & (x - 1)) == 0
-	}
-}
-
 // NewFs contstructs an FsDrive from the path, container:path
 func NewFs(name, path string) (fs.Fs, error) {
-	if !isPowerOfTwo(int64(chunkSize)) {
-		return nil, fmt.Errorf("drive: chunk size %v isn't a power of two", chunkSize)
-	}
-	if chunkSize < 256*1024 {
-		return nil, fmt.Errorf("drive: chunk size can't be less than 256k - was %v", chunkSize)
-	}
-
 	t, err := driveAuth.NewTransport(name)
 	if err != nil {
 		return nil, err
@@ -332,15 +217,10 @@ func NewFs(name, path string) (fs.Fs, error) {
 	}
 
 	f := &FsDrive{
-		root:      root,
-		dirCache:  newDirCache(),
-		pacer:     make(chan struct{}, 1),
-		sleepTime: minSleep,
+		root:     root,
+		dirCache: newDirCache(),
 	}
 
-	// Put the first pacing token in
-	f.pacer <- struct{}{}
-
 	// Create a new authorized Drive client.
 	f.client = t.Client()
 	f.svc, err = drive.New(f.client)
@@ -349,15 +229,15 @@ func NewFs(name, path string) (fs.Fs, error) {
 	}
 
 	// Read About so we know the root path
-	f.call(&err, func() {
-		f.about, err = f.svc.About.Get().Do()
-	})
+	f.about, err = f.svc.About.Get().Do()
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't read info about Drive: %s", err)
 	}
 
-	// Find the Id of the true root and clear everything
-	f.resetRoot()
+	// Find the Id of the root directory and the Id of its parent
+	f.rootId = f.about.RootFolderId
+	// Put the root directory in
+	f.dirCache.Put("", f.rootId)
 	// Find the current root
 	err = f.findRoot(false)
 	if err != nil {
@@ -371,7 +251,7 @@ func NewFs(name, path string) (fs.Fs, error) {
 			// No root so return old f
 			return f, nil
 		}
-		obj, err := newF.newFsObjectWithInfoErr(remote, nil)
+		obj, err := newF.newFsObjectWithInfo(remote, nil)
 		if err != nil {
 			// File doesn't exist so return old f
 			return f, nil
@@ -384,7 +264,7 @@ func NewFs(name, path string) (fs.Fs, error) {
 }
 
 // Return an FsObject from a path
-func (f *FsDrive) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) {
+func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
 	fs := &FsObjectDrive{
 		drive:  f,
 		remote: remote,
@@ -404,8 +284,8 @@ func (f *FsDrive) newFsObjectWithInfoErr(remote string, info *drive.File) (fs.Ob
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object {
-	fs, _ := f.newFsObjectWithInfoErr(remote, info)
+func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
+	fs, _ := f.newFsObjectWithInfo(remote, info)
 	// Errors have already been logged
 	return fs
 }
@@ -414,7 +294,7 @@ func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object
 //
 // May return nil if an error occurred
 func (f *FsDrive) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
 // Path should be directory path either "" or "path/"
@@ -438,7 +318,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan
 		} else {
 			// If item has no MD5 sum it isn't stored on drive, so ignore it
 			if item.Md5Checksum != "" {
-				if fs := f.newFsObjectWithInfo(path+item.Title, item); fs != nil {
+				if fs := f.NewFsObjectWithInfo(path+item.Title, item); fs != nil {
 					out <- fs
 				}
 			}
@@ -488,7 +368,7 @@ func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) err
 			// fmt.Printf("file %s %s %s\n", path, item.Title, item.Id)
 			// If item has no MD5 sum it isn't stored on drive, so ignore it
 			if item.Md5Checksum != "" {
-				if fs := f.newFsObjectWithInfo(path, item); fs != nil {
+				if fs := f.NewFsObjectWithInfo(path, item); fs != nil {
 					out <- fs
 				}
 			}
@@ -611,18 +491,15 @@ func (f *FsDrive) _findDir(path string, create bool) (pathId string, err error)
 		if create {
 			// fmt.Println("Making", path)
 			// Define the metadata for the directory we are going to create.
-			createInfo := &drive.File{
+			info := &drive.File{
 				Title:       leaf,
 				Description: leaf,
 				MimeType:    driveFolderType,
 				Parents:     []*drive.ParentReference{{Id: pathId}},
 			}
-			var info *drive.File
-			f.call(&err, func() {
-				info, err = f.svc.Files.Insert(createInfo).Do()
-			})
+			info, err := f.svc.Files.Insert(info).Do()
 			if err != nil {
-				return pathId, fmt.Errorf("Failed to make directory: %v", err)
+				return pathId, fmt.Errorf("Failed to make directory")
 			}
 			pathId = info.Id
 		} else {
@@ -660,20 +537,6 @@ func (f *FsDrive) findRoot(create bool) error {
 	return nil
 }
 
-// Resets the root directory to the absolute root and clears the dirCache
-func (f *FsDrive) resetRoot() {
-	f.findRootLock.Lock()
-	defer f.findRootLock.Unlock()
-	f.foundRoot = false
-	f.dirCache.Flush()
-
-	// Put the true root in
-	f.rootId = f.about.RootFolderId
-
-	// Put the root directory in
-	f.dirCache.Put("", f.rootId)
-}
-
 // Walk the path returning a channel of FsObjects
 func (f *FsDrive) List() fs.ObjectsChan {
 	out := make(fs.ObjectsChan, fs.Config.Checkers)
@@ -682,16 +545,16 @@ func (f *FsDrive) List() fs.ObjectsChan {
 		err := f.findRoot(false)
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't find root: %s", err)
+			log.Printf("Couldn't find root: %s", err)
 		} else {
-			if f.root == "" && *driveFullList {
+			if *driveFullList {
 				err = f.listDirFull(f.rootId, "", out)
 			} else {
 				err = f.listDirRecursive(f.rootId, "", out)
 			}
 			if err != nil {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "List failed: %s", err)
+				log.Printf("List failed: %s", err)
 			}
 		}
 	}()
@@ -706,7 +569,7 @@ func (f *FsDrive) ListDir() fs.DirChan {
 		err := f.findRoot(false)
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't find root: %s", err)
+			log.Printf("Couldn't find root: %s", err)
 		} else {
 			_, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool {
 				dir := &fs.Dir{
@@ -714,19 +577,43 @@ func (f *FsDrive) ListDir() fs.DirChan {
 					Bytes: -1,
 					Count: -1,
 				}
-				dir.When, _ = time.Parse(timeFormatIn, item.ModifiedDate)
+				dir.When, _ = time.Parse(RFC3339In, item.ModifiedDate)
 				out <- dir
 				return false
 			})
 			if err != nil {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "ListDir failed: %s", err)
+				log.Printf("ListDir failed: %s", err)
 			}
 		}
 	}()
 	return out
 }
 
+// seekWrapper wraps an io.Reader with a basic Seek for
+// code.google.com/p/google-api-go-client/googleapi
+// to detect the length (see getReaderSize function)
+type seekWrapper struct {
+	in   io.Reader
+	size int64
+}
+
+// Read bytes from the object - see io.Reader
+func (file *seekWrapper) Read(p []byte) (n int, err error) {
+	return file.in.Read(p)
+}
+
+// Seek - minimal implementation for Google Drive's length detection
+func (file *seekWrapper) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case os.SEEK_CUR:
+		return 0, nil
+	case os.SEEK_END:
+		return file.size, nil
+	}
+	return 0, nil
+}
+
 // Put the object
 //
 // This assumes that the object doesn't not already exists - if you
@@ -746,33 +633,27 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
 		return o, fmt.Errorf("Couldn't find or make directory: %s", err)
 	}
 
+	// Guess the mime type
+	mimeType := mime.TypeByExtension(path.Ext(o.remote))
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+	modifiedDate := modTime.Format(RFC3339Out)
+
 	// Define the metadata for the file we are going to create.
-	createInfo := &drive.File{
+	info := &drive.File{
 		Title:        leaf,
 		Description:  leaf,
 		Parents:      []*drive.ParentReference{{Id: directoryId}},
-		MimeType:     fs.MimeType(o),
-		ModifiedDate: modTime.Format(timeFormatOut),
+		MimeType:     mimeType,
+		ModifiedDate: modifiedDate,
 	}
 
-	var info *drive.File
-	if size == 0 || size < int64(driveUploadCutoff) {
-		// Make the API request to upload metadata and file data.
-		// Don't retry, return a retry error instead
-		f.beginCall()
-		info, err = f.svc.Files.Insert(createInfo).Media(in).Do()
-		if f.endCall(err) {
-			return o, fs.RetryErrorf("Upload failed - retry: %s", err)
-		}
-		if err != nil {
-			return o, fmt.Errorf("Upload failed: %s", err)
-		}
-	} else {
-		// Upload the file in chunks
-		info, err = f.Upload(in, size, createInfo.MimeType, createInfo, remote)
-		if err != nil {
-			return o, err
-		}
+	// Make the API request to upload metadata and file data.
+	in = &seekWrapper{in: in, size: size}
+	info, err = f.svc.Files.Insert(info).Media(in).Do()
+	if err != nil {
+		return o, fmt.Errorf("Upload failed: %s", err)
 	}
 	o.setMetaData(info)
 	return o, nil
@@ -791,10 +672,7 @@ func (f *FsDrive) Rmdir() error {
 	if err != nil {
 		return err
 	}
-	var children *drive.ChildList
-	f.call(&err, func() {
-		children, err = f.svc.Children.List(f.rootId).MaxResults(10).Do()
-	})
+	children, err := f.svc.Children.List(f.rootId).MaxResults(10).Do()
 	if err != nil {
 		return err
 	}
@@ -803,18 +681,11 @@ func (f *FsDrive) Rmdir() error {
 	}
 	// Delete the directory if it isn't the root
 	if f.root != "" {
-		f.call(&err, func() {
-			if *driveUseTrash {
-				_, err = f.svc.Files.Trash(f.rootId).Do()
-			} else {
-				err = f.svc.Files.Delete(f.rootId).Do()
-			}
-		})
+		err = f.svc.Files.Delete(f.rootId).Do()
 		if err != nil {
 			return err
 		}
 	}
-	f.resetRoot()
 	return nil
 }
 
@@ -836,14 +707,8 @@ func (f *FsDrive) Purge() error {
 	if err != nil {
 		return err
 	}
-	f.call(&err, func() {
-		if *driveUseTrash {
-			_, err = f.svc.Files.Trash(f.rootId).Do()
-		} else {
-			err = f.svc.Files.Delete(f.rootId).Do()
-		}
-	})
-	f.resetRoot()
+	err = f.svc.Files.Delete(f.rootId).Do()
+	f.dirCache.Flush()
 	if err != nil {
 		return err
 	}
@@ -930,7 +795,7 @@ func (o *FsObjectDrive) ModTime() time.Time {
 		fs.Log(o, "Failed to read metadata: %s", err)
 		return time.Now()
 	}
-	modTime, err := time.Parse(timeFormatIn, o.modifiedDate)
+	modTime, err := time.Parse(RFC3339In, o.modifiedDate)
 	if err != nil {
 		fs.Log(o, "Failed to read mtime from object: %s", err)
 		return time.Now()
@@ -943,25 +808,19 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
 	err := o.readMetaData()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to read metadata: %s", err)
+		fs.Log(o, "Failed to read metadata: %s", err)
 		return
 	}
 	// New metadata
-	updateInfo := &drive.File{
-		ModifiedDate: modTime.Format(timeFormatOut),
+	info := &drive.File{
+		ModifiedDate: modTime.Format(RFC3339Out),
 	}
 	// Set modified date
-	var info *drive.File
-	o.drive.call(&err, func() {
-		info, err = o.drive.svc.Files.Update(o.id, updateInfo).SetModifiedDate(true).Do()
-	})
+	_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
-		return
+		fs.Log(o, "Failed to update remote mtime: %s", err)
 	}
-	// Update info from read data
-	o.setMetaData(info)
 }
 
 // Is this object storable
@@ -971,23 +830,17 @@ func (o *FsObjectDrive) Storable() bool {
 
 // Open an object for read
 func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
-	if o.url == "" {
-		return nil, fmt.Errorf("Forbidden to download - check sharing permission")
-	}
 	req, err := http.NewRequest("GET", o.url, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("User-Agent", fs.UserAgent)
-	var res *http.Response
-	o.drive.call(&err, func() {
-		res, err = o.drive.client.Do(req)
-	})
+	res, err := o.drive.client.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	if res.StatusCode != 200 {
-		_ = res.Body.Close() // ignore error
+		res.Body.Close()
 		return nil, fmt.Errorf("Bad response: %d: %s", res.StatusCode, res.Status)
 	}
 	return res.Body, nil
@@ -999,30 +852,16 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
 //
 // The new object may have been created if an error is returned
 func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
-	updateInfo := &drive.File{
+	info := &drive.File{
 		Id:           o.id,
-		ModifiedDate: modTime.Format(timeFormatOut),
+		ModifiedDate: modTime.Format(RFC3339Out),
 	}
 
 	// Make the API request to upload metadata and file data.
-	var err error
-	var info *drive.File
-	if size == 0 || size < int64(driveUploadCutoff) {
-		// Don't retry, return a retry error instead
-		o.drive.beginCall()
-		info, err = o.drive.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do()
-		if o.drive.endCall(err) {
-			return fs.RetryErrorf("Update failed - retry: %s", err)
-		}
-		if err != nil {
-			return fmt.Errorf("Update failed: %s", err)
-		}
-	} else {
-		// Upload the file in chunks
-		info, err = o.drive.Upload(in, size, fs.MimeType(o), updateInfo, o.remote)
-		if err != nil {
-			return err
-		}
+	in = &seekWrapper{in: in, size: size}
+	info, err := o.drive.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
+	if err != nil {
+		return fmt.Errorf("Update failed: %s", err)
 	}
 	o.setMetaData(info)
 	return nil
@@ -1030,15 +869,7 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
 
 // Remove an object
 func (o *FsObjectDrive) Remove() error {
-	var err error
-	o.drive.call(&err, func() {
-		if *driveUseTrash {
-			_, err = o.drive.svc.Files.Trash(o.id).Do()
-		} else {
-			err = o.drive.svc.Files.Delete(o.id).Do()
-		}
-	})
-	return err
+	return o.drive.svc.Files.Delete(o.id).Do()
 }
 
 // Check the interfaces are satisfied

drive/drive_test.go

@@ -1,53 +0,0 @@
-// Test Drive filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package drive_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/drive"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*drive.FsObjectDrive)(nil))
-	fstests.RemoteName = "TestDrive:"
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

drive/upload.go

@@ -1,246 +0,0 @@
-// Upload for drive
-//
-// Docs
-// Resumable upload: https://developers.google.com/drive/web/manage-uploads#resumable
-// Best practices: https://developers.google.com/drive/web/manage-uploads#best-practices
-// Files insert: https://developers.google.com/drive/v2/reference/files/insert
-// Files update: https://developers.google.com/drive/v2/reference/files/update
-//
-// This contains code adapted from google.golang.org/api (C) the GO AUTHORS
-
-package drive
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strconv"
-
-	"github.com/ncw/rclone/fs"
-	"google.golang.org/api/drive/v2"
-	"google.golang.org/api/googleapi"
-)
-
-const (
-	// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
-	statusResumeIncomplete = 308
-
-	// Number of times to try each chunk
-	maxTries = 10
-)
-
-// resumableUpload is used by the generated APIs to provide resumable uploads.
-// It is not used by developers directly.
-type resumableUpload struct {
-	f      *FsDrive
-	remote string
-	// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
-	URI string
-	// Media is the object being uploaded.
-	Media io.Reader
-	// MediaType defines the media type, e.g. "image/jpeg".
-	MediaType string
-	// ContentLength is the full size of the object being uploaded.
-	ContentLength int64
-	// Return value
-	ret *drive.File
-}
-
-// Upload the io.Reader in of size bytes with contentType and info
-func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *drive.File, remote string) (*drive.File, error) {
-	fileId := info.Id
-	var body io.Reader = nil
-	body, err := googleapi.WithoutDataWrapper.JSONReader(info)
-	if err != nil {
-		return nil, err
-	}
-	params := make(url.Values)
-	params.Set("alt", "json")
-	params.Set("uploadType", "resumable")
-	urls := "https://www.googleapis.com/upload/drive/v2/files"
-	method := "POST"
-	if fileId != "" {
-		params.Set("setModifiedDate", "true")
-		urls += "/{fileId}"
-		method = "PUT"
-	}
-	urls += "?" + params.Encode()
-	req, _ := http.NewRequest(method, urls, body)
-	googleapi.Expand(req.URL, map[string]string{
-		"fileId": fileId,
-	})
-	req.Header.Set("Content-Type", "application/json; charset=UTF-8")
-	req.Header.Set("X-Upload-Content-Type", contentType)
-	req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
-	req.Header.Set("User-Agent", fs.UserAgent)
-	var res *http.Response
-	f.call(&err, func() {
-		res, err = f.client.Do(req)
-		if err == nil {
-			defer googleapi.CloseBody(res)
-			err = googleapi.CheckResponse(res)
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-	loc := res.Header.Get("Location")
-	rx := &resumableUpload{
-		f:             f,
-		remote:        remote,
-		URI:           loc,
-		Media:         in,
-		MediaType:     contentType,
-		ContentLength: size,
-	}
-	return rx.Upload()
-}
-
-// Make an http.Request for the range passed in
-func (rx *resumableUpload) makeRequest(start int64, body []byte) *http.Request {
-	reqSize := int64(len(body))
-	req, _ := http.NewRequest("POST", rx.URI, bytes.NewBuffer(body))
-	req.ContentLength = reqSize
-	if reqSize != 0 {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
-	} else {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
-	}
-	req.Header.Set("Content-Type", rx.MediaType)
-	req.Header.Set("User-Agent", fs.UserAgent)
-	return req
-}
-
-// rangeRE matches the transfer status response from the server. $1 is
-// the last byte index uploaded.
-var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
-
-// Query drive for the amount transferred so far
-//
-// If error is nil, then start should be valid
-func (rx *resumableUpload) transferStatus() (start int64, err error) {
-	req := rx.makeRequest(0, nil)
-	res, err := rx.f.client.Do(req)
-	if err != nil {
-		return 0, err
-	}
-	defer googleapi.CloseBody(res)
-	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
-		return rx.ContentLength, nil
-	}
-	if res.StatusCode != statusResumeIncomplete {
-		err = googleapi.CheckResponse(res)
-		if err != nil {
-			return 0, err
-		}
-		return 0, fmt.Errorf("unexpected http return code %v", res.StatusCode)
-	}
-	Range := res.Header.Get("Range")
-	if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 {
-		start, err = strconv.ParseInt(m[1], 10, 64)
-		if err == nil {
-			return start, nil
-		}
-	}
-	return 0, fmt.Errorf("unable to parse range %q", Range)
-}
-
-// Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
-func (rx *resumableUpload) transferChunk(start int64, body []byte) (int, error) {
-	req := rx.makeRequest(start, body)
-	res, err := rx.f.client.Do(req)
-	if err != nil {
-		return 599, err
-	}
-	defer googleapi.CloseBody(res)
-	if res.StatusCode == statusResumeIncomplete {
-		return res.StatusCode, nil
-	}
-	err = googleapi.CheckResponse(res)
-	if err != nil {
-		return res.StatusCode, err
-	}
-
-	// When the entire file upload is complete, the server
-	// responds with an HTTP 201 Created along with any metadata
-	// associated with this resource. If this request had been
-	// updating an existing entity rather than creating a new one,
-	// the HTTP response code for a completed upload would have
-	// been 200 OK.
-	//
-	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionaly on
-	// StatusCode
-	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
-		return 598, err
-	}
-
-	return res.StatusCode, nil
-}
-
-// Upload uploads the chunks from the input
-// It retries each chunk maxTries times (with a pause of uploadPause between attempts).
-func (rx *resumableUpload) Upload() (*drive.File, error) {
-	start := int64(0)
-	buf := make([]byte, chunkSize)
-	var StatusCode int
-	for start < rx.ContentLength {
-		reqSize := rx.ContentLength - start
-		if reqSize >= int64(chunkSize) {
-			reqSize = int64(chunkSize)
-		} else {
-			buf = buf[:reqSize]
-		}
-
-		// Read the chunk
-		_, err := io.ReadFull(rx.Media, buf)
-		if err != nil {
-			return nil, err
-		}
-
-		// Transfer the chunk
-		for try := 1; try <= maxTries; try++ {
-			fs.Debug(rx.remote, "Sending chunk %d length %d, %d/%d", start, reqSize, try, maxTries)
-			rx.f.beginCall()
-			StatusCode, err = rx.transferChunk(start, buf)
-			rx.f.endCall(err)
-			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
-				goto success
-			}
-			fs.Debug(rx.remote, "Retrying chunk %d/%d, code=%d, err=%v", try, maxTries, StatusCode, err)
-		}
-		fs.Debug(rx.remote, "Failed to send chunk")
-		return nil, fs.RetryErrorf("Chunk upload failed - retry: code=%d, err=%v", StatusCode, err)
-	success:
-
-		start += reqSize
-	}
-	// Resume or retry uploads that fail due to connection interruptions or
-	// any 5xx errors, including:
-	//
-	// 500 Internal Server Error
-	// 502 Bad Gateway
-	// 503 Service Unavailable
-	// 504 Gateway Timeout
-	//
-	// Use an exponential backoff strategy if any 5xx server error is
-	// returned when resuming or retrying upload requests. These errors can
-	// occur if a server is getting overloaded. Exponential backoff can help
-	// alleviate these kinds of problems during periods of high volume of
-	// requests or heavy network traffic.  Other kinds of requests should not
-	// be handled by exponential backoff but you can still retry a number of
-	// them. When retrying these requests, limit the number of times you
-	// retry them. For example your code could limit to ten retries or less
-	// before reporting an error.
-	//
-	// Handle 404 Not Found errors when doing resumable uploads by starting
-	// the entire upload over from the beginning.
-	if rx.ret == nil {
-		return nil, fs.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode)
-	}
-	return rx.ret, nil
-}

dropbox/dropbox.go

@@ -6,26 +6,17 @@ Limitations of dropbox
 
 File system is case insensitive
 
+The datastore is limited to 100,000 records which therefore is the
+limit of the number of files that rclone can use on dropbox.
+
+FIXME only open datastore if we need it?
+
 FIXME Getting this sometimes
 Failed to copy: Upload failed: invalid character '<' looking for beginning of value
 This is a JSON decode error - from Update / UploadByChunk
 - Caused by 500 error from dropbox
 - See https://github.com/stacktic/dropbox/issues/1
 - Possibly confusing dropbox with excess concurrency?
-
-FIXME implement timeouts - need to get "github.com/stacktic/dropbox"
-and hence "golang.org/x/oauth2" which uses DefaultTransport unless it
-is set in the context passed into .Client()
-
-func (db *Dropbox) client() *http.Client {
-	return db.config.Client(oauth2.NoContext, db.token)
-}
-
-// HTTPClient is the context key to use with golang.org/x/net/context's
-// WithValue function to associate an *http.Client value with a context.
-var HTTPClient ContextKey
-
-So pass in a context with HTTPClient set...
 */
 
 import (
@@ -33,10 +24,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/ncw/rclone/fs"
@@ -45,10 +36,17 @@ import (
 
 // Constants
 const (
-	rcloneAppKey    = "5jcck7diasz0rqy"
-	rcloneAppSecret = "1n9m04y2zx7bf26"
-	uploadChunkSize = 64 * 1024                    // chunk size for upload
-	metadataLimit   = dropbox.MetadataLimitDefault // max items to fetch at once
+	rcloneAppKey     = "5jcck7diasz0rqy"
+	rcloneAppSecret  = "1n9m04y2zx7bf26"
+	uploadChunkSize  = 64 * 1024                    // chunk size for upload
+	metadataLimit    = dropbox.MetadataLimitDefault // max items to fetch at once
+	datastoreName    = "rclone"
+	tableName        = "metadata"
+	md5sumField      = "md5sum"
+	mtimeField       = "mtime"
+	maxCommitRetries = 5
+	RFC3339In        = time.RFC3339
+	RFC3339Out       = "2006-01-02T15:04:05.000000000Z07:00"
 )
 
 // Register with Fs
@@ -56,7 +54,7 @@ func init() {
 	fs.Register(&fs.FsInfo{
 		Name:   "dropbox",
 		NewFs:  NewFs,
-		Config: configHelper,
+		Config: Config,
 		Options: []fs.Option{{
 			Name: "app_key",
 			Help: "Dropbox App Key - leave blank to use rclone's.",
@@ -68,7 +66,7 @@ func init() {
 }
 
 // Configuration helper - called after the user has put in the defaults
-func configHelper(name string) {
+func Config(name string) {
 	// See if already have a token
 	token := fs.ConfigFile.MustValue(name, "token")
 	if token != "" {
@@ -99,19 +97,24 @@ func configHelper(name string) {
 
 // FsDropbox represents a remote dropbox server
 type FsDropbox struct {
-	db             *dropbox.Dropbox // the connection to the dropbox server
-	root           string           // the path we are working on
-	slashRoot      string           // root with "/" prefix, lowercase
-	slashRootSlash string           // root with "/" prefix and postfix, lowercase
+	db               *dropbox.Dropbox // the connection to the dropbox server
+	root             string           // the path we are working on
+	slashRoot        string           // root with "/" prefix
+	slashRootSlash   string           // root with "/" prefix and postix
+	datastoreManager *dropbox.DatastoreManager
+	datastore        *dropbox.Datastore
+	table            *dropbox.Table
+	datastoreMutex   sync.Mutex // lock this when using the datastore
+	datastoreErr     error      // pending errors on the datastore
 }
 
 // FsObjectDropbox describes a dropbox object
 type FsObjectDropbox struct {
-	dropbox     *FsDropbox // what this object is part of
-	remote      string     // The remote path
-	bytes       int64      // size of the object
-	modTime     time.Time  // time it was last modified
-	hasMetadata bool       // metadata is valid
+	dropbox *FsDropbox // what this object is part of
+	remote  string     // The remote path
+	md5sum  string     // md5sum of the object
+	bytes   int64      // size of the object
+	modTime time.Time  // time it was last modified
 }
 
 // ------------------------------------------------------------
@@ -153,6 +156,12 @@ func NewFs(name, root string) (fs.Fs, error) {
 	// Authorize the client
 	db.SetAccessToken(token)
 
+	// Make a db to store rclone metadata in
+	f.datastoreManager = db.NewDatastoreManager()
+
+	// Open the datastore in the background
+	go f.openDataStore()
+
 	// See if the root is actually an object
 	entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
 	if err == nil && !entry.IsDir {
@@ -173,19 +182,39 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Sets root in f
 func (f *FsDropbox) setRoot(root string) {
 	f.root = strings.Trim(root, "/")
-	lowerCaseRoot := strings.ToLower(f.root)
-
-	f.slashRoot = "/" + lowerCaseRoot
+	f.slashRoot = "/" + f.root
 	f.slashRootSlash = f.slashRoot
-	if lowerCaseRoot != "" {
+	if f.root != "" {
 		f.slashRootSlash += "/"
 	}
 }
 
+// Opens the datastore in f
+func (f *FsDropbox) openDataStore() {
+	f.datastoreMutex.Lock()
+	defer f.datastoreMutex.Unlock()
+	fs.Debug(f, "Open rclone datastore")
+	// Open the rclone datastore
+	var err error
+	f.datastore, err = f.datastoreManager.OpenDatastore(datastoreName)
+	if err != nil {
+		fs.Log(f, "Failed to open datastore: %v", err)
+		f.datastoreErr = err
+		return
+	}
+
+	// Get the table we are using
+	f.table, err = f.datastore.GetTable(tableName)
+	if err != nil {
+		fs.Log(f, "Failed to open datastore table: %v", err)
+		f.datastoreErr = err
+		return
+	}
+	fs.Debug(f, "Open rclone datastore finished")
+}
+
 // Return an FsObject from a path
-//
-// May return nil if an error occurred
-func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
+func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) {
 	o := &FsObjectDropbox{
 		dropbox: f,
 		remote:  remote,
@@ -196,49 +225,49 @@ func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.O
 		err := o.readEntryAndSetMetadata()
 		if err != nil {
 			// logged already fs.Debug("Failed to read info: %s", err)
-			return nil
+			return nil, err
 		}
 	}
-	return o
+	return o, nil
+}
+
+// Return an FsObject from a path
+//
+// May return nil if an error occurred
+func (f *FsDropbox) NewFsObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
+	fs, _ := f.newFsObjectWithInfo(remote, info)
+	// Errors have already been logged
+	return fs
 }
 
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
 func (f *FsDropbox) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
-// Strips the root off path and returns it
-func (f *FsDropbox) stripRoot(path string) *string {
-	lowercase := strings.ToLower(path)
-
-	if !strings.HasPrefix(lowercase, f.slashRootSlash) {
-		fs.Stats.Error()
-		fs.ErrorLog(f, "Path '%s' is not under root '%s'", path, f.slashRootSlash)
-		return nil
+// Strips the root off entry and returns it
+func (f *FsDropbox) stripRoot(entry *dropbox.Entry) string {
+	path := entry.Path
+	if strings.HasPrefix(path, f.slashRootSlash) {
+		path = path[len(f.slashRootSlash):]
 	}
-
-	stripped := path[len(f.slashRootSlash):]
-	return &stripped
+	return path
 }
 
 // Walk the root returning a channel of FsObjects
 func (f *FsDropbox) list(out fs.ObjectsChan) {
-	// Track path component case, it could be different for entries coming from DropBox API
-	// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
-	// and https://github.com/ncw/rclone/issues/53
-	nameTree := NewNameTree()
 	cursor := ""
 	for {
 		deltaPage, err := f.db.Delta(cursor, f.slashRoot)
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't list: %s", err)
+			fs.Log(f, "Couldn't list: %s", err)
 			break
 		} else {
 			if deltaPage.Reset && cursor != "" {
-				fs.ErrorLog(f, "Unexpected reset during listing - try again")
+				fs.Log(f, "Unexpected reset during listing - try again")
 				fs.Stats.Error()
 				break
 			}
@@ -248,58 +277,29 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
 				entry := deltaEntry.Entry
 				if entry == nil {
 					// This notifies of a deleted object
+					fs.Debug(f, "Deleting metadata for %q", deltaEntry.Path)
+					key := metadataKey(deltaEntry.Path) // Path is lowercased
+					err := f.deleteMetadata(key)
+					if err != nil {
+						fs.Debug(f, "Failed to delete metadata for %q", deltaEntry.Path)
+						// Don't accumulate Error here
+					}
+
 				} else {
-					if len(entry.Path) <= 1 || entry.Path[0] != '/' {
-						fs.Stats.Error()
-						fs.ErrorLog(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path)
-						continue
-					}
-
-					lastSlashIndex := strings.LastIndex(entry.Path, "/")
-
-					var parentPath string
-					if lastSlashIndex == 0 {
-						parentPath = ""
-					} else {
-						parentPath = entry.Path[1:lastSlashIndex]
-					}
-					lastComponent := entry.Path[lastSlashIndex+1:]
-
 					if entry.IsDir {
-						nameTree.PutCaseCorrectDirectoryName(parentPath, lastComponent)
+						// ignore directories
 					} else {
-						parentPathCorrectCase := nameTree.GetPathWithCorrectCase(parentPath)
-						if parentPathCorrectCase != nil {
-							path := f.stripRoot(*parentPathCorrectCase + "/" + lastComponent)
-							if path == nil {
-								// an error occurred and logged by stripRoot
-								continue
-							}
-
-							out <- f.newFsObjectWithInfo(*path, entry)
-						} else {
-							nameTree.PutFile(parentPath, lastComponent, entry)
-						}
+						path := f.stripRoot(entry)
+						out <- f.NewFsObjectWithInfo(path, entry)
 					}
 				}
 			}
 			if !deltaPage.HasMore {
 				break
 			}
-			cursor = deltaPage.Cursor.Cursor
+			cursor = deltaPage.Cursor
 		}
 	}
-
-	walkFunc := func(caseCorrectFilePath string, entry *dropbox.Entry) {
-		path := f.stripRoot("/" + caseCorrectFilePath)
-		if path == nil {
-			// an error occurred and logged by stripRoot
-			return
-		}
-
-		out <- f.newFsObjectWithInfo(*path, entry)
-	}
-	nameTree.WalkFiles(f.root, walkFunc)
 }
 
 // Walk the path returning a channel of FsObjects
@@ -320,21 +320,15 @@ func (f *FsDropbox) ListDir() fs.DirChan {
 		entry, err := f.db.Metadata(f.root, true, false, "", "", metadataLimit)
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't list directories in root: %s", err)
+			fs.Log(f, "Couldn't list directories in root: %s", err)
 		} else {
 			for i := range entry.Contents {
 				entry := &entry.Contents[i]
 				if entry.IsDir {
-					name := f.stripRoot(entry.Path)
-					if name == nil {
-						// an error occurred and logged by stripRoot
-						continue
-					}
-
 					out <- &fs.Dir{
-						Name:  *name,
+						Name:  f.stripRoot(entry),
 						When:  time.Time(entry.ClientMtime),
-						Bytes: entry.Bytes,
+						Bytes: int64(entry.Bytes),
 						Count: -1,
 					}
 				}
@@ -398,8 +392,8 @@ func (f *FsDropbox) Rmdir() error {
 }
 
 // Return the precision
-func (f *FsDropbox) Precision() time.Duration {
-	return fs.ModTimeNotSupported
+func (fs *FsDropbox) Precision() time.Duration {
+	return time.Nanosecond
 }
 
 // Purge deletes all the files and the container
@@ -408,11 +402,81 @@ func (f *FsDropbox) Precision() time.Duration {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *FsDropbox) Purge() error {
+	// Delete metadata first
+	var wg sync.WaitGroup
+	to_be_deleted := f.List()
+	wg.Add(fs.Config.Transfers)
+	for i := 0; i < fs.Config.Transfers; i++ {
+		go func() {
+			defer wg.Done()
+			for dst := range to_be_deleted {
+				o := dst.(*FsObjectDropbox)
+				o.deleteMetadata()
+			}
+		}()
+	}
+	wg.Wait()
+
 	// Let dropbox delete the filesystem tree
 	_, err := f.db.Delete(f.slashRoot)
 	return err
 }
 
+// Tries the transaction in fn then calls commit, repeating until retry limit
+//
+// Holds datastore mutex while in progress
+func (f *FsDropbox) transaction(fn func() error) error {
+	f.datastoreMutex.Lock()
+	defer f.datastoreMutex.Unlock()
+	if f.datastoreErr != nil {
+		return f.datastoreErr
+	}
+	var err error
+	for i := 1; i <= maxCommitRetries; i++ {
+		err = fn()
+		if err != nil {
+			return err
+		}
+
+		err = f.datastore.Commit()
+		if err == nil {
+			break
+		}
+		fs.Debug(f, "Retrying transaction %d/%d", i, maxCommitRetries)
+	}
+	if err != nil {
+		return fmt.Errorf("Failed to commit metadata changes: %s", err)
+	}
+	return nil
+}
+
+// Deletes the medadata associated with this key
+func (f *FsDropbox) deleteMetadata(key string) error {
+	return f.transaction(func() error {
+		record, err := f.table.Get(key)
+		if err != nil {
+			return fmt.Errorf("Couldn't get record: %s", err)
+		}
+		if record == nil {
+			return nil
+		}
+		record.DeleteRecord()
+		return nil
+	})
+}
+
+// Reads the record attached to key
+//
+// Holds datastore mutex while in progress
+func (f *FsDropbox) readRecord(key string) (*dropbox.Record, error) {
+	f.datastoreMutex.Lock()
+	defer f.datastoreMutex.Unlock()
+	if f.datastoreErr != nil {
+		return nil, f.datastoreErr
+	}
+	return f.table.Get(key)
+}
+
 // ------------------------------------------------------------
 
 // Return the parent Fs
@@ -434,8 +498,33 @@ func (o *FsObjectDropbox) Remote() string {
 }
 
 // Md5sum returns the Md5sum of an object returning a lowercase hex string
+//
+// FIXME has to download the file!
 func (o *FsObjectDropbox) Md5sum() (string, error) {
-	return "", nil
+	if o.md5sum != "" {
+		return o.md5sum, nil
+	}
+	err := o.readMetaData()
+	if err != nil {
+		fs.Log(o, "Failed to read metadata: %s", err)
+		return "", fmt.Errorf("Failed to read metadata: %s", err)
+
+	}
+
+	// For pre-existing files which have no md5sum can read it and set it?
+
+	// in, err := o.Open()
+	// if err != nil {
+	// 	return "", err
+	// }
+	// defer in.Close()
+	// hash := md5.New()
+	// _, err = io.Copy(hash, in)
+	// if err != nil {
+	// 	return "", err
+	// }
+	// o.md5sum = fmt.Sprintf("%x", hash.Sum(nil))
+	return o.md5sum, nil
 }
 
 // Size returns the size of an object in bytes
@@ -447,9 +536,8 @@ func (o *FsObjectDropbox) Size() int64 {
 //
 // This isn't a complete set of metadata and has an inacurate date
 func (o *FsObjectDropbox) setMetadataFromEntry(info *dropbox.Entry) {
-	o.bytes = info.Bytes
+	o.bytes = int64(info.Bytes)
 	o.modTime = time.Time(info.ClientMtime)
-	o.hasMetadata = true
 }
 
 // Reads the entry from dropbox
@@ -485,9 +573,7 @@ func (o *FsObjectDropbox) remotePath() string {
 func metadataKey(path string) string {
 	// NB File system is case insensitive
 	path = strings.ToLower(path)
-	hash := md5.New()
-	_, _ = hash.Write([]byte(path))
-	return fmt.Sprintf("%x", hash.Sum(nil))
+	return fmt.Sprintf("%x", md5.Sum([]byte(path)))
 }
 
 // Returns the key for the metadata database
@@ -497,11 +583,58 @@ func (o *FsObjectDropbox) metadataKey() string {
 
 // readMetaData gets the info if it hasn't already been fetched
 func (o *FsObjectDropbox) readMetaData() (err error) {
-	if o.hasMetadata {
+	if o.md5sum != "" {
 		return nil
 	}
+
+	// fs.Debug(o, "Reading metadata from datastore")
+	record, err := o.dropbox.readRecord(o.metadataKey())
+	if err != nil {
+		fs.Debug(o, "Couldn't read metadata: %s", err)
+		record = nil
+	}
+
+	if record != nil {
+		// Read md5sum
+		md5sumInterface, ok, err := record.Get(md5sumField)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			fs.Debug(o, "Couldn't find md5sum in record")
+		} else {
+			md5sum, ok := md5sumInterface.(string)
+			if !ok {
+				fs.Debug(o, "md5sum not a string")
+			} else {
+				o.md5sum = md5sum
+			}
+		}
+
+		// read mtime
+		mtimeInterface, ok, err := record.Get(mtimeField)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			fs.Debug(o, "Couldn't find mtime in record")
+		} else {
+			mtime, ok := mtimeInterface.(string)
+			if !ok {
+				fs.Debug(o, "mtime not a string")
+			} else {
+				modTime, err := time.Parse(RFC3339In, mtime)
+				if err != nil {
+					return err
+				}
+				o.modTime = modTime
+			}
+		}
+	}
+
 	// Last resort
-	return o.readEntryAndSetMetadata()
+	o.readEntryAndSetMetadata()
+	return nil
 }
 
 // ModTime returns the modification time of the object
@@ -517,12 +650,57 @@ func (o *FsObjectDropbox) ModTime() time.Time {
 	return o.modTime
 }
 
+// Sets the modification time of the local fs object into the record
+// FIXME if we don't set md5sum what will that do?
+func (o *FsObjectDropbox) setModTimeAndMd5sum(modTime time.Time, md5sum string) error {
+	key := o.metadataKey()
+	// fs.Debug(o, "Writing metadata to datastore")
+	return o.dropbox.transaction(func() error {
+		record, err := o.dropbox.table.GetOrInsert(key)
+		if err != nil {
+			return fmt.Errorf("Couldn't read record: %s", err)
+		}
+
+		if md5sum != "" {
+			err = record.Set(md5sumField, md5sum)
+			if err != nil {
+				return fmt.Errorf("Couldn't set md5sum record: %s", err)
+			}
+		}
+
+		if !modTime.IsZero() {
+			mtime := modTime.Format(RFC3339Out)
+			err := record.Set(mtimeField, mtime)
+			if err != nil {
+				return fmt.Errorf("Couldn't set mtime record: %s", err)
+			}
+		}
+
+		return nil
+	})
+}
+
+// Deletes the medadata associated with this file
+//
+// It logs any errors
+func (o *FsObjectDropbox) deleteMetadata() {
+	fs.Debug(o, "Deleting metadata from datastore")
+	err := o.dropbox.deleteMetadata(o.metadataKey())
+	if err != nil {
+		fs.Log(o, "Error deleting metadata: %v", err)
+		fs.Stats.Error()
+	}
+}
+
 // Sets the modification time of the local fs object
 //
 // Commits the datastore
 func (o *FsObjectDropbox) SetModTime(modTime time.Time) {
-	// FIXME not implemented
-	return
+	err := o.setModTimeAndMd5sum(modTime, "")
+	if err != nil {
+		fs.Stats.Error()
+		fs.Log(o, err.Error())
+	}
 }
 
 // Is this object storable
@@ -542,16 +720,22 @@ func (o *FsObjectDropbox) Open() (in io.ReadCloser, err error) {
 //
 // The new object may have been created if an error is returned
 func (o *FsObjectDropbox) Update(in io.Reader, modTime time.Time, size int64) error {
-	entry, err := o.dropbox.db.UploadByChunk(ioutil.NopCloser(in), uploadChunkSize, o.remotePath(), true, "")
+	// Calculate md5sum as we upload it
+	hash := md5.New()
+	rc := &readCloser{in: io.TeeReader(in, hash)}
+	entry, err := o.dropbox.db.UploadByChunk(rc, uploadChunkSize, o.remotePath(), true, "")
 	if err != nil {
 		return fmt.Errorf("Upload failed: %s", err)
 	}
 	o.setMetadataFromEntry(entry)
-	return nil
+
+	md5sum := fmt.Sprintf("%x", hash.Sum(nil))
+	return o.setModTimeAndMd5sum(modTime, md5sum)
 }
 
 // Remove an object
 func (o *FsObjectDropbox) Remove() error {
+	o.deleteMetadata()
 	_, err := o.dropbox.db.Delete(o.remotePath())
 	return err
 }

dropbox/dropbox_test.go

@@ -1,53 +0,0 @@
-// Test Dropbox filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package dropbox_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/dropbox"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*dropbox.FsObjectDropbox)(nil))
-	fstests.RemoteName = "TestDropbox:"
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

dropbox/nametree.go

@@ -1,179 +0,0 @@
-package dropbox
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/ncw/rclone/fs"
-	"github.com/stacktic/dropbox"
-	"strings"
-)
-
-type NameTreeNode struct {
-	// Map from lowercase directory name to tree node
-	Directories map[string]*NameTreeNode
-
-	// Map from file name (case sensitive) to dropbox entry
-	Files map[string]*dropbox.Entry
-
-	// Empty string if exact case is unknown or root node
-	CaseCorrectName string
-}
-
-// ------------------------------------------------------------
-
-func newNameTreeNode(caseCorrectName string) *NameTreeNode {
-	return &NameTreeNode{
-		CaseCorrectName: caseCorrectName,
-		Directories:     make(map[string]*NameTreeNode),
-		Files:           make(map[string]*dropbox.Entry),
-	}
-}
-
-func NewNameTree() *NameTreeNode {
-	return newNameTreeNode("")
-}
-
-func (tree *NameTreeNode) String() string {
-	if len(tree.CaseCorrectName) == 0 {
-		return "NameTreeNode/<root>"
-	} else {
-		return fmt.Sprintf("NameTreeNode/%q", tree.CaseCorrectName)
-	}
-}
-
-func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode {
-	if len(path) == 0 {
-		// no lookup required, just return root
-		return tree
-	}
-
-	current := tree
-	for _, component := range strings.Split(path, "/") {
-		if len(component) == 0 {
-			fs.Stats.Error()
-			fs.ErrorLog(tree, "getTreeNode: path component is empty (full path %q)", path)
-			return nil
-		}
-
-		lowercase := strings.ToLower(component)
-
-		lookup := current.Directories[lowercase]
-		if lookup == nil {
-			lookup = newNameTreeNode("")
-			current.Directories[lowercase] = lookup
-		}
-
-		current = lookup
-	}
-
-	return current
-}
-
-func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) {
-	if len(caseCorrectDirectoryName) == 0 {
-		fs.Stats.Error()
-		fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath)
-		return
-	}
-
-	node := tree.getTreeNode(parentPath)
-	if node == nil {
-		return
-	}
-
-	lowerCaseDirectoryName := strings.ToLower(caseCorrectDirectoryName)
-	directory := node.Directories[lowerCaseDirectoryName]
-	if directory == nil {
-		directory = newNameTreeNode(caseCorrectDirectoryName)
-		node.Directories[lowerCaseDirectoryName] = directory
-	} else {
-		if len(directory.CaseCorrectName) > 0 {
-			fs.Stats.Error()
-			fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath)
-			return
-		}
-
-		directory.CaseCorrectName = caseCorrectDirectoryName
-	}
-}
-
-func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) {
-	node := tree.getTreeNode(parentPath)
-	if node == nil {
-		return
-	}
-
-	if node.Files[caseCorrectFileName] != nil {
-		fs.Stats.Error()
-		fs.ErrorLog(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath)
-		return
-	}
-
-	node.Files[caseCorrectFileName] = dropboxEntry
-}
-
-func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string {
-	if path == "" {
-		empty := ""
-		return &empty
-	}
-
-	var result bytes.Buffer
-
-	current := tree
-	for _, component := range strings.Split(path, "/") {
-		if component == "" {
-			fs.Stats.Error()
-			fs.ErrorLog(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path)
-			return nil
-		}
-
-		lowercase := strings.ToLower(component)
-
-		current = current.Directories[lowercase]
-		if current == nil || current.CaseCorrectName == "" {
-			return nil
-		}
-
-		result.WriteString("/")
-		result.WriteString(current.CaseCorrectName)
-	}
-
-	resultString := result.String()
-	return &resultString
-}
-
-type NameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry)
-
-func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFileWalkFunc) {
-	var prefix string
-	if currentPath == "" {
-		prefix = ""
-	} else {
-		prefix = currentPath + "/"
-	}
-
-	for name, entry := range tree.Files {
-		walkFunc(prefix+name, entry)
-	}
-
-	for lowerCaseName, directory := range tree.Directories {
-		caseCorrectName := directory.CaseCorrectName
-		if caseCorrectName == "" {
-			fs.Stats.Error()
-			fs.ErrorLog(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath)
-			continue
-		}
-
-		directory.walkFilesRec(prefix+caseCorrectName, walkFunc)
-	}
-}
-
-func (tree *NameTreeNode) WalkFiles(rootPath string, walkFunc NameTreeFileWalkFunc) {
-	node := tree.getTreeNode(rootPath)
-	if node == nil {
-		return
-	}
-
-	node.walkFilesRec(rootPath, walkFunc)
-}

dropbox/nametree_test.go

@@ -1,124 +0,0 @@
-package dropbox_test
-
-import (
-	"github.com/ncw/rclone/dropbox"
-	"github.com/ncw/rclone/fs"
-	dropboxapi "github.com/stacktic/dropbox"
-	"testing"
-)
-
-func assert(t *testing.T, shouldBeTrue bool, failMessage string) {
-	if !shouldBeTrue {
-		t.Fatal(failMessage)
-	}
-}
-
-func TestPutCaseCorrectDirectoryName(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutCaseCorrectDirectoryName("a/b", "C")
-
-	assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty")
-
-	a := tree.Directories["a"]
-	assert(t, a.CaseCorrectName == "", "CaseCorrectName at 'a' should be empty")
-
-	b := a.Directories["b"]
-	assert(t, b.CaseCorrectName == "", "CaseCorrectName at 'a/b' should be empty")
-
-	c := b.Directories["c"]
-	assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'a/b/c' should be 'C'")
-
-	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutCaseCorrectDirectoryName("/a", "C")
-	tree.PutCaseCorrectDirectoryName("b/", "C")
-	tree.PutCaseCorrectDirectoryName("a//b", "C")
-
-	assert(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported")
-}
-
-func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutCaseCorrectDirectoryName("", "C")
-
-	c := tree.Directories["c"]
-	assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'")
-
-	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestGetPathWithCorrectCase(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutCaseCorrectDirectoryName("a", "C")
-	assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
-
-	tree.PutCaseCorrectDirectoryName("", "A")
-	assert(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'")
-
-	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalk(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-	tree.PutCaseCorrectDirectoryName("", "A")
-
-	numCalled := 0
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
-		assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
-		assert(t, entry.Path == "xxx", "entry.Path should be xxx")
-		numCalled++
-	}
-	tree.WalkFiles("", walkFunc)
-
-	assert(t, numCalled == 1, "walk func should be called only once")
-
-	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalkWithPrefix(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-	tree.PutCaseCorrectDirectoryName("", "A")
-
-	numCalled := 0
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
-		assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
-		assert(t, entry.Path == "xxx", "entry.Path should be xxx")
-		numCalled++
-	}
-	tree.WalkFiles("A", walkFunc)
-
-	assert(t, numCalled == 1, "walk func should be called only once")
-
-	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalkIncompleteTree(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := dropbox.NewNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
-		t.Fatal("Should not be called")
-	}
-	tree.WalkFiles("", walkFunc)
-
-	assert(t, fs.Stats.GetErrors() == errors+1, "One error should be reported")
-}

fs/accounting.go

@@ -10,24 +10,13 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"github.com/tsenart/tb"
 )
 
 // Globals
 var (
-	Stats       = NewStats()
-	tokenBucket *tb.Bucket
+	Stats = NewStats()
 )
 
-// Start the token bucket if necessary
-func startTokenBucket() {
-	if bwLimit > 0 {
-		tokenBucket = tb.NewBucket(int64(bwLimit), 100*time.Millisecond)
-		Log(nil, "Starting bandwidth limiter at %vBytes/s", &bwLimit)
-	}
-}
-
 // Stringset holds some strings
 type StringSet map[string]bool
 
@@ -124,16 +113,6 @@ func (s *StatsInfo) GetErrors() int64 {
 	return s.errors
 }
 
-// ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
-func (s *StatsInfo) ResetCounters() {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-	s.bytes = 0
-	s.errors = 0
-	s.checks = 0
-	s.transfers = 0
-}
-
 // Errored returns whether there have been any errors
 func (s *StatsInfo) Errored() bool {
 	s.lock.RLock()
@@ -163,13 +142,6 @@ func (s *StatsInfo) DoneChecking(o Object) {
 	s.checks += 1
 }
 
-// GetTransfers reads the number of transfers
-func (s *StatsInfo) GetTransfers() int64 {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-	return s.transfers
-}
-
 // Transferring adds a transfer into the stats
 func (s *StatsInfo) Transferring(o Object) {
 	s.lock.Lock()
@@ -187,12 +159,6 @@ func (s *StatsInfo) DoneTransferring(o Object) {
 
 // Account limits and accounts for one transfer
 type Account struct {
-	// The mutex is to make sure Read() and Close() aren't called
-	// concurrently.  Unfortunately the persistent connection loop
-	// in http transport calls Read() after Do() returns on
-	// CancelRequest so this race can happen when it apparently
-	// shouldn't.
-	mu    sync.Mutex
 	in    io.ReadCloser
 	bytes int64
 }
@@ -206,25 +172,17 @@ func NewAccount(in io.ReadCloser) *Account {
 
 // Read bytes from the object - see io.Reader
 func (file *Account) Read(p []byte) (n int, err error) {
-	file.mu.Lock()
-	defer file.mu.Unlock()
 	n, err = file.in.Read(p)
 	file.bytes += int64(n)
 	Stats.Bytes(int64(n))
 	if err == io.EOF {
 		// FIXME Do something?
 	}
-	// Limit the transfer speed if required
-	if tokenBucket != nil {
-		tokenBucket.Wait(int64(n))
-	}
 	return
 }
 
 // Close the object
 func (file *Account) Close() error {
-	file.mu.Lock()
-	defer file.mu.Unlock()
 	// FIXME do something?
 	return file.in.Close()
 }

fs/config.go

@@ -6,8 +6,6 @@ import (
 	"bufio"
 	"fmt"
 	"log"
-	"math"
-	"net/http"
 	"os"
 	"os/user"
 	"path"
@@ -17,7 +15,6 @@ import (
 	"time"
 
 	"github.com/Unknwon/goconfig"
-	"github.com/mreiferson/go-httpclient"
 	"github.com/ogier/pflag"
 )
 
@@ -25,8 +22,6 @@ const (
 	configFileName = ".rclone.conf"
 )
 
-type SizeSuffix int64
-
 // Global
 var (
 	// Config file
@@ -38,151 +33,34 @@ var (
 	// Global config
 	Config = &ConfigInfo{}
 	// Flags
-	verbose        = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
-	quiet          = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
-	modifyWindow   = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
-	checkers       = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
-	transfers      = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
-	configFile     = pflag.StringP("config", "", ConfigPath, "Config file.")
-	checkSum       = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
-	sizeOnly       = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
-	dryRun         = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
-	connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
-	timeout        = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
-	bwLimit        SizeSuffix
+	verbose      = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
+	quiet        = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
+	modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
+	checkers     = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
+	transfers    = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
+	configFile   = pflag.StringP("config", "", ConfigPath, "Config file.")
+	dryRun       = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
 )
 
-func init() {
-	pflag.VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix k|M|G")
-}
-
-// Turn SizeSuffix into a string
-func (x SizeSuffix) String() string {
-	scaled := float64(0)
-	suffix := ""
-	switch {
-	case x == 0:
-		return "0"
-	case x < 1024*1024:
-		scaled = float64(x) / 1024
-		suffix = "k"
-	case x < 1024*1024*1024:
-		scaled = float64(x) / 1024 / 1024
-		suffix = "M"
-	default:
-		scaled = float64(x) / 1024 / 1024 / 1024
-		suffix = "G"
-	}
-	if math.Floor(scaled) == scaled {
-		return fmt.Sprintf("%.0f%s", scaled, suffix)
-	}
-	return fmt.Sprintf("%.3f%s", scaled, suffix)
-}
-
-// Set a SizeSuffix
-func (x *SizeSuffix) Set(s string) error {
-	if len(s) == 0 {
-		return fmt.Errorf("Empty string")
-	}
-	suffix := s[len(s)-1]
-	suffixLen := 1
-	var multiplier float64
-	switch suffix {
-	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
-		suffixLen = 0
-		multiplier = 1 << 10
-	case 'k', 'K':
-		multiplier = 1 << 10
-	case 'm', 'M':
-		multiplier = 1 << 20
-	case 'g', 'G':
-		multiplier = 1 << 30
-	default:
-		return fmt.Errorf("Bad suffix %q", suffix)
-	}
-	s = s[:len(s)-suffixLen]
-	value, err := strconv.ParseFloat(s, 64)
-	if err != nil {
-		return err
-	}
-	if value < 0 {
-		return fmt.Errorf("Size can't be negative %q", s)
-	}
-	value *= multiplier
-	*x = SizeSuffix(value)
-	return nil
-}
-
-// Check it satisfies the interface
-var _ pflag.Value = (*SizeSuffix)(nil)
-
 // Filesystem config options
 type ConfigInfo struct {
-	Verbose        bool
-	Quiet          bool
-	DryRun         bool
-	CheckSum       bool
-	SizeOnly       bool
-	ModifyWindow   time.Duration
-	Checkers       int
-	Transfers      int
-	ConnectTimeout time.Duration // Connect timeout
-	Timeout        time.Duration // Data channel timeout
-}
-
-// Transport returns an http.RoundTripper with the correct timeouts
-func (ci *ConfigInfo) Transport() http.RoundTripper {
-	return &httpclient.Transport{
-		Proxy:               http.ProxyFromEnvironment,
-		MaxIdleConnsPerHost: ci.Checkers + ci.Transfers + 1,
-
-		// ConnectTimeout, if non-zero, is the maximum amount of time a dial will wait for
-		// a connect to complete.
-		ConnectTimeout: ci.ConnectTimeout,
-
-		// ResponseHeaderTimeout, if non-zero, specifies the amount of
-		// time to wait for a server's response headers after fully
-		// writing the request (including its body, if any). This
-		// time does not include the time to read the response body.
-		ResponseHeaderTimeout: ci.Timeout,
-
-		// RequestTimeout, if non-zero, specifies the amount of time for the entire
-		// request to complete (including all of the above timeouts + entire response body).
-		// This should never be less than the sum total of the above two timeouts.
-		//RequestTimeout: NOT SET,
-
-		// ReadWriteTimeout, if non-zero, will set a deadline for every Read and
-		// Write operation on the request connection.
-		ReadWriteTimeout: ci.Timeout,
-	}
-}
-
-// Transport returns an http.Client with the correct timeouts
-func (ci *ConfigInfo) Client() *http.Client {
-	return &http.Client{
-		Transport: ci.Transport(),
-	}
+	Verbose      bool
+	Quiet        bool
+	DryRun       bool
+	ModifyWindow time.Duration
+	Checkers     int
+	Transfers    int
 }
 
 // Find the config directory
 func configHome() string {
 	// Find users home directory
 	usr, err := user.Current()
-	if err == nil {
-		return usr.HomeDir
+	if err != nil {
+		log.Printf("Couldn't find home directory: %v", err)
+		return ""
 	}
-	// Fall back to reading $HOME - work around user.Current() not
-	// working for cross compiled binaries on OSX.
-	// https://github.com/golang/go/issues/6376
-	home := os.Getenv("HOME")
-	if home != "" {
-		return home
-	}
-	log.Printf("Couldn't find home directory or read HOME environment variable.")
-	log.Printf("Defaulting to storing config in current directory.")
-	log.Printf("Use -config flag to workaround.")
-	log.Printf("Error was: %v", err)
-	return ""
+	return usr.HomeDir
 }
 
 // Loads the config file
@@ -196,10 +74,6 @@ func LoadConfig() {
 	Config.Checkers = *checkers
 	Config.Transfers = *transfers
 	Config.DryRun = *dryRun
-	Config.Timeout = *timeout
-	Config.ConnectTimeout = *connectTimeout
-	Config.CheckSum = *checkSum
-	Config.SizeOnly = *sizeOnly
 
 	ConfigPath = *configFile
 
@@ -207,15 +81,12 @@ func LoadConfig() {
 	var err error
 	ConfigFile, err = goconfig.LoadConfigFile(ConfigPath)
 	if err != nil {
-		log.Printf("Failed to load config file %v - using defaults: %v", ConfigPath, err)
+		log.Printf("Failed to load config file %v - using defaults", ConfigPath)
 		ConfigFile, err = goconfig.LoadConfigFile(os.DevNull)
 		if err != nil {
 			log.Fatalf("Failed to read null config file: %v", err)
 		}
 	}
-
-	// Start the token bucket limiter
-	startTokenBucket()
 }
 
 // Save configuration file.
@@ -449,20 +320,9 @@ func EditConfig() {
 			name := ChooseRemote()
 			EditRemote(name)
 		case 'n':
-		nameLoop:
-			for {
-				fmt.Printf("name> ")
-				name := ReadLine()
-				switch {
-				case name == "":
-					fmt.Printf("Can't use empty name\n")
-				case isDriveLetter(name):
-					fmt.Printf("Can't use %q as it can be confused a drive letter\n", name)
-				default:
-					NewRemote(name)
-					break nameLoop
-				}
-			}
+			fmt.Printf("name> ")
+			name := ReadLine()
+			NewRemote(name)
 		case 'd':
 			name := ChooseRemote()
 			DeleteRemote(name)

fs/config_test.go

@@ -1,57 +0,0 @@
-package fs
-
-import "testing"
-
-func TestSizeSuffixString(t *testing.T) {
-	for _, test := range []struct {
-		in   float64
-		want string
-	}{
-		{0, "0"},
-		{102, "0.100k"},
-		{1024, "1k"},
-		{1024 * 1024, "1M"},
-		{1024 * 1024 * 1024, "1G"},
-		{10 * 1024 * 1024 * 1024, "10G"},
-		{10.1 * 1024 * 1024 * 1024, "10.100G"},
-	} {
-		ss := SizeSuffix(test.in)
-		got := ss.String()
-		if test.want != got {
-			t.Errorf("Want %v got %v", test.want, got)
-		}
-	}
-}
-
-func TestSizeSuffixSet(t *testing.T) {
-	for i, test := range []struct {
-		in   string
-		want int64
-		err  bool
-	}{
-		{"0", 0, false},
-		{"0.1k", 102, false},
-		{"0.1", 102, false},
-		{"1K", 1024, false},
-		{"1", 1024, false},
-		{"2.5", 1024 * 2.5, false},
-		{"1M", 1024 * 1024, false},
-		{"1.g", 1024 * 1024 * 1024, false},
-		{"10G", 10 * 1024 * 1024 * 1024, false},
-		{"", 0, true},
-		{"1p", 0, true},
-		{"1.p", 0, true},
-		{"1p", 0, true},
-		{"-1K", 0, true},
-	} {
-		ss := SizeSuffix(0)
-		err := ss.Set(test.in)
-		if (err != nil) != test.err {
-			t.Errorf("%d: Expecting error %v but got error %v", i, test.err, err)
-		}
-		got := int64(ss)
-		if test.want != got {
-			t.Errorf("%d: Want %v got %v", i, test.want, got)
-		}
-	}
-}

fs/driveletter.go

@@ -1,12 +0,0 @@
-// +build !windows
-
-package fs
-
-// isDriveLetter returns a bool indicating whether name is a valid
-// Windows drive letter
-//
-// On non windows platforms we don't have drive letters so we always
-// return false
-func isDriveLetter(name string) bool {
-	return false
-}

fs/driveletter_windows.go

@@ -1,13 +0,0 @@
-// +build windows
-
-package fs
-
-// isDriveLetter returns a bool indicating whether name is a valid
-// Windows drive letter
-func isDriveLetter(name string) bool {
-	if len(name) != 1 {
-		return false
-	}
-	c := name[0]
-	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-}

fs/fs.go

@@ -1,11 +1,11 @@
-// Generic file system interface for rclone object storage systems
+// File system interface
+
 package fs
 
 import (
 	"fmt"
 	"io"
 	"log"
-	"path/filepath"
 	"regexp"
 	"time"
 )
@@ -14,16 +14,12 @@ import (
 const (
 	// User agent for Fs which can set it
 	UserAgent = "rclone/" + Version
-	// Very large precision value to show mod time isn't supported
-	ModTimeNotSupported = 100 * 365 * 24 * time.Hour
 )
 
 // Globals
 var (
 	// Filesystem registry
 	fsRegistry []*FsInfo
-	// Error returned by NewFs if not found in config file
-	NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
 )
 
 // Filesystem info
@@ -83,13 +79,9 @@ type Fs interface {
 	Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
 
 	// Make the directory (container, bucket)
-	//
-	// Shouldn't return an error if it already exists
 	Mkdir() error
 
 	// Remove the directory (container, bucket) if empty
-	//
-	// Return an error if it doesn't exist or isn't empty
 	Rmdir() error
 
 	// Precision of the ModTimes in this Fs
@@ -109,11 +101,9 @@ type Object interface {
 	Remote() string
 
 	// Md5sum returns the md5 checksum of the file
-	// If no Md5sum is available it returns ""
 	Md5sum() (string, error)
 
 	// ModTime returns the modification date of the file
-	// It should return a best guess if one isn't available
 	ModTime() time.Time
 
 	// SetModTime sets the metadata on the object to set the modification date
@@ -141,40 +131,9 @@ type Purger interface {
 	//
 	// Implement this if you have a way of deleting all the files
 	// quicker than just running Remove() on the result of List()
-	//
-	// Return an error if it doesn't exist
 	Purge() error
 }
 
-// An optional interface for error as to whether the operation should be retried
-//
-// This should be returned from Update or Put methods as required
-type Retry interface {
-	error
-	Retry() bool
-}
-
-// A type of error
-type retryError string
-
-// Error interface
-func (r retryError) Error() string {
-	return string(r)
-}
-
-// Retry interface
-func (r retryError) Retry() bool {
-	return true
-}
-
-// Check interface
-var _ Retry = retryError("")
-
-// RetryErrorf makes an error which indicates it would like to be retried
-func RetryErrorf(format string, a ...interface{}) error {
-	return retryError(fmt.Sprintf(format, a...))
-}
-
 // A channel of Objects
 type ObjectsChan chan Object
 
@@ -200,6 +159,9 @@ type Dir struct {
 // A channel of Dir objects
 type DirChan chan *Dir
 
+// Pattern to match a url
+var matcher = regexp.MustCompile(`^([\w_-]+):(.*)$`)
+
 // Finds a FsInfo object for the name passed in
 //
 // Services are looked up in the config file
@@ -212,43 +174,34 @@ func Find(name string) (*FsInfo, error) {
 	return nil, fmt.Errorf("Didn't find filing system for %q", name)
 }
 
-// Pattern to match an rclone url
-var matcher = regexp.MustCompile(`^([\w_-]+):(.*)$`)
-
 // NewFs makes a new Fs object from the path
 //
-// The path is of the form remote:path
+// The path is of the form service://path
 //
-// Remotes are looked up in the config file.  If the remote isn't
-// found then NotFoundInConfigFile will be returned.
-//
-// On Windows avoid single character remote names as they can be mixed
-// up with drive letters.
+// Services are looked up in the config file
 func NewFs(path string) (Fs, error) {
 	parts := matcher.FindStringSubmatch(path)
 	fsName, configName, fsPath := "local", "local", path
-	if parts != nil && !isDriveLetter(parts[1]) {
+	if parts != nil {
 		configName, fsPath = parts[1], parts[2]
 		var err error
 		fsName, err = ConfigFile.GetValue(configName, "type")
 		if err != nil {
-			return nil, NotFoundInConfigFile
+			return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
 		}
 	}
 	fs, err := Find(fsName)
 	if err != nil {
 		return nil, err
 	}
-	// change native directory separators to / if there are any
-	fsPath = filepath.ToSlash(fsPath)
 	return fs.NewFs(configName, fsPath)
 }
 
 // Outputs log for object
 func OutputLog(o interface{}, text string, args ...interface{}) {
 	description := ""
-	if o != nil {
-		description = fmt.Sprintf("%v: ", o)
+	if x, ok := o.(fmt.Stringer); ok {
+		description = x.String() + ": "
 	}
 	out := fmt.Sprintf(text, args...)
 	log.Print(description + out)
@@ -268,12 +221,6 @@ func Log(o interface{}, text string, args ...interface{}) {
 	}
 }
 
-// Write error log output for this Object or Fs
-// Unconditionally logs a message regardless of Config.Quiet or Config.Verbose
-func ErrorLog(o interface{}, text string, args ...interface{}) {
-	OutputLog(o, text, args...)
-}
-
 // checkClose is a utility function used to check the return from
 // Close in a defer statement.
 func checkClose(c io.Closer, err *error) {

fs/operations.go

@@ -4,11 +4,8 @@ package fs
 
 import (
 	"fmt"
-	"io"
-	"mime"
-	"path"
+	"log"
 	"sync"
-	"time"
 )
 
 // Work out modify window for fses passed in - sets Config.ModifyWindow
@@ -27,14 +24,6 @@ func CalculateModifyWindow(fs ...Fs) {
 	Debug(fs[0], "Modify window is %s\n", Config.ModifyWindow)
 }
 
-// Md5sumsEqual checks to see if src == dst, but ignores empty strings
-func Md5sumsEqual(src, dst string) bool {
-	if src == "" || dst == "" {
-		return true
-	}
-	return src == dst
-}
-
 // Check the two files to see if the MD5sums are the same
 //
 // May return an error which will already have been logged
@@ -44,34 +33,33 @@ func CheckMd5sums(src, dst Object) (bool, error) {
 	srcMd5, err := src.Md5sum()
 	if err != nil {
 		Stats.Error()
-		ErrorLog(src, "Failed to calculate src md5: %s", err)
+		Log(src, "Failed to calculate src md5: %s", err)
 		return false, err
 	}
 	dstMd5, err := dst.Md5sum()
 	if err != nil {
 		Stats.Error()
-		ErrorLog(dst, "Failed to calculate dst md5: %s", err)
+		Log(dst, "Failed to calculate dst md5: %s", err)
 		return false, err
 	}
 	// Debug("Src MD5 %s", srcMd5)
 	// Debug("Dst MD5 %s", obj.Hash)
-	return Md5sumsEqual(srcMd5, dstMd5), nil
+	return srcMd5 == dstMd5, nil
 }
 
 // Checks to see if the src and dst objects are equal by looking at
 // size, mtime and MD5SUM
 //
 // If the src and dst size are different then it is considered to be
-// not equal.  If --size-only is in effect then this is the only check
-// that is done.
+// not equal.
 //
 // If the size is the same and the mtime is the same then it is
-// considered to be equal.  This check is skipped if using --checksum.
+// considered to be equal.  This is the heuristic rsync uses when
+// not using --checksum.
 //
-// If the size is the same and mtime is different, unreadable or
-// --checksum is set and the MD5SUM is the same then the file is
-// considered to be equal.  In this case the mtime on the dst is
-// updated if --checksum is not set.
+// If the size is the same and and mtime is different or unreadable
+// and the MD5SUM is the same then the file is considered to be equal.
+// In this case the mtime on the dst is updated.
 //
 // Otherwise the file is considered to be not equal including if there
 // were errors reading info.
@@ -80,24 +68,17 @@ func Equal(src, dst Object) bool {
 		Debug(src, "Sizes differ")
 		return false
 	}
-	if Config.SizeOnly {
-		Debug(src, "Sizes identical")
-		return true
-	}
 
-	var srcModTime time.Time
-	if !Config.CheckSum {
-		// Size the same so check the mtime
-		srcModTime = src.ModTime()
-		dstModTime := dst.ModTime()
-		dt := dstModTime.Sub(srcModTime)
-		ModifyWindow := Config.ModifyWindow
-		if dt >= ModifyWindow || dt <= -ModifyWindow {
-			Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
-		} else {
-			Debug(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, ModifyWindow)
-			return true
-		}
+	// Size the same so check the mtime
+	srcModTime := src.ModTime()
+	dstModTime := dst.ModTime()
+	dt := dstModTime.Sub(srcModTime)
+	ModifyWindow := Config.ModifyWindow
+	if dt >= ModifyWindow || dt <= -ModifyWindow {
+		Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
+	} else {
+		Debug(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, ModifyWindow)
+		return true
 	}
 
 	// mtime is unreadable or different but size is the same so
@@ -108,39 +89,24 @@ func Equal(src, dst Object) bool {
 		return false
 	}
 
-	if !Config.CheckSum {
-		// Size and MD5 the same but mtime different so update the
-		// mtime of the dst object here
-		dst.SetModTime(srcModTime)
-	}
+	// Size and MD5 the same but mtime different so update the
+	// mtime of the dst object here
+	dst.SetModTime(srcModTime)
 
 	Debug(src, "Size and MD5SUM of src and dst objects identical")
 	return true
 }
 
-// Returns a guess at the mime type from the extension
-func MimeType(o Object) string {
-	mimeType := mime.TypeByExtension(path.Ext(o.Remote()))
-	if mimeType == "" {
-		mimeType = "application/octet-stream"
-	}
-	return mimeType
-}
-
 // Used to remove a failed copy
-//
-// Returns whether the file was succesfully removed or not
-func removeFailedCopy(dst Object) bool {
-	if dst == nil {
-		return false
+func removeFailedCopy(dst Object) {
+	if dst != nil {
+		Debug(dst, "Removing failed copy")
+		removeErr := dst.Remove()
+		if removeErr != nil {
+			Stats.Error()
+			Log(dst, "Failed to remove failed copy: %s", removeErr)
+		}
 	}
-	Debug(dst, "Removing failed copy")
-	removeErr := dst.Remove()
-	if removeErr != nil {
-		Debug(dst, "Failed to remove failed copy: %s", removeErr)
-		return false
-	}
-	return true
 }
 
 // Copy src object to dst or f if nil
@@ -149,20 +115,16 @@ func removeFailedCopy(dst Object) bool {
 // call Copy() with dst nil on a pre-existing file then some filing
 // systems (eg Drive) may duplicate the file.
 func Copy(f Fs, dst, src Object) {
-	const maxTries = 10
-	tries := 0
-	doUpdate := dst != nil
-tryAgain:
 	in0, err := src.Open()
 	if err != nil {
 		Stats.Error()
-		ErrorLog(src, "Failed to open: %s", err)
+		Log(src, "Failed to open: %s", err)
 		return
 	}
 	in := NewAccount(in0) // account the transfer
 
 	var actionTaken string
-	if doUpdate {
+	if dst != nil {
 		actionTaken = "Copied (updated existing)"
 		err = dst.Update(in, src.ModTime(), src.Size())
 	} else {
@@ -170,23 +132,12 @@ tryAgain:
 		dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
 	}
 	inErr := in.Close()
-	// Retry if err returned a retry error
-	if r, ok := err.(Retry); ok && r.Retry() && tries < maxTries {
-		tries++
-		Log(src, "Received error: %v - retrying %d/%d", err, tries, maxTries)
-		if removeFailedCopy(dst) {
-			// If we removed dst, then nil it out and note we are not updating
-			dst = nil
-			doUpdate = false
-		}
-		goto tryAgain
-	}
 	if err == nil {
 		err = inErr
 	}
 	if err != nil {
 		Stats.Error()
-		ErrorLog(src, "Failed to copy: %s", err)
+		Log(src, "Failed to copy: %s", err)
 		removeFailedCopy(dst)
 		return
 	}
@@ -195,29 +146,27 @@ tryAgain:
 	if src.Size() != dst.Size() {
 		Stats.Error()
 		err = fmt.Errorf("Corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
-		ErrorLog(dst, "%s", err)
+		Log(dst, "%s", err)
 		removeFailedCopy(dst)
 		return
 	}
 
 	// Verify md5sums are the same after transfer - ignoring blank md5sums
-	if !Config.SizeOnly {
-		srcMd5sum, md5sumErr := src.Md5sum()
+	srcMd5sum, md5sumErr := src.Md5sum()
+	if md5sumErr != nil {
+		Stats.Error()
+		Log(src, "Failed to read md5sum: %s", md5sumErr)
+	} else if srcMd5sum != "" {
+		dstMd5sum, md5sumErr := dst.Md5sum()
 		if md5sumErr != nil {
 			Stats.Error()
-			ErrorLog(src, "Failed to read md5sum: %s", md5sumErr)
-		} else if srcMd5sum != "" {
-			dstMd5sum, md5sumErr := dst.Md5sum()
-			if md5sumErr != nil {
-				Stats.Error()
-				ErrorLog(dst, "Failed to read md5sum: %s", md5sumErr)
-			} else if !Md5sumsEqual(srcMd5sum, dstMd5sum) {
-				Stats.Error()
-				err = fmt.Errorf("Corrupted on transfer: md5sums differ %q vs %q", srcMd5sum, dstMd5sum)
-				ErrorLog(dst, "%s", err)
-				removeFailedCopy(dst)
-				return
-			}
+			Log(dst, "Failed to read md5sum: %s", md5sumErr)
+		} else if dstMd5sum != "" && srcMd5sum != dstMd5sum {
+			Stats.Error()
+			err = fmt.Errorf("Corrupted on transfer: md5sums differ %q vs %q", srcMd5sum, dstMd5sum)
+			Log(dst, "%s", err)
+			removeFailedCopy(dst)
+			return
 		}
 	}
 
@@ -276,10 +225,12 @@ func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
 func DeleteFiles(to_be_deleted ObjectsChan) {
 	var wg sync.WaitGroup
 	wg.Add(Config.Transfers)
+	var fs Fs
 	for i := 0; i < Config.Transfers; i++ {
 		go func() {
 			defer wg.Done()
 			for dst := range to_be_deleted {
+				fs = dst.Fs()
 				if Config.DryRun {
 					Debug(dst, "Not deleting as --dry-run")
 				} else {
@@ -288,7 +239,7 @@ func DeleteFiles(to_be_deleted ObjectsChan) {
 					Stats.DoneChecking(dst)
 					if err != nil {
 						Stats.Error()
-						ErrorLog(dst, "Couldn't delete: %s", err)
+						Log(dst, "Couldn't delete: %s", err)
 					} else {
 						Debug(dst, "Deleted")
 					}
@@ -296,22 +247,9 @@ func DeleteFiles(to_be_deleted ObjectsChan) {
 			}
 		}()
 	}
-	Log(nil, "Waiting for deletions to finish")
-	wg.Wait()
-}
 
-// Read a map of Object.Remote to Object for the given Fs
-func readFilesMap(fs Fs) map[string]Object {
-	files := make(map[string]Object)
-	for o := range fs.List() {
-		remote := o.Remote()
-		if _, ok := files[remote]; !ok {
-			files[remote] = o
-		} else {
-			Log(o, "Duplicate file detected")
-		}
-	}
-	return files
+	Log(fs, "Waiting for deletions to finish")
+	wg.Wait()
 }
 
 // Syncs fsrc into fdst
@@ -328,7 +266,10 @@ func Sync(fdst, fsrc Fs, Delete bool) error {
 
 	// Read the destination files first
 	// FIXME could do this in parallel and make it use less memory
-	delFiles := readFilesMap(fdst)
+	delFiles := make(map[string]Object)
+	for dst := range fdst.List() {
+		delFiles[dst.Remote()] = dst
+	}
 
 	// Read source files checking them off against dest files
 	to_be_checked := make(ObjectPairChan, Config.Transfers)
@@ -393,33 +334,35 @@ func Check(fdst, fsrc Fs) error {
 
 	// Read the destination files first
 	// FIXME could do this in parallel and make it use less memory
-	dstFiles := readFilesMap(fdst)
+	dstFiles := make(map[string]Object)
+	for dst := range fdst.List() {
+		dstFiles[dst.Remote()] = dst
+	}
 
 	// Read the source files checking them against dstFiles
 	// FIXME could do this in parallel and make it use less memory
-	srcFiles := readFilesMap(fsrc)
-
-	// Move all the common files into commonFiles and delete then
-	// from srcFiles and dstFiles
+	srcFiles := make(map[string]Object)
 	commonFiles := make(map[string][]Object)
-	for remote, src := range srcFiles {
+	for src := range fsrc.List() {
+		remote := src.Remote()
 		if dst, ok := dstFiles[remote]; ok {
 			commonFiles[remote] = []Object{dst, src}
-			delete(srcFiles, remote)
 			delete(dstFiles, remote)
+		} else {
+			srcFiles[remote] = src
 		}
 	}
 
 	Log(fdst, "%d files not in %v", len(dstFiles), fsrc)
 	for _, dst := range dstFiles {
 		Stats.Error()
-		ErrorLog(dst, "File not in %v", fsrc)
+		Log(dst, "File not in %v", fsrc)
 	}
 
 	Log(fsrc, "%d files not in %s", len(srcFiles), fdst)
 	for _, src := range srcFiles {
 		Stats.Error()
-		ErrorLog(src, "File not in %v", fdst)
+		Log(src, "File not in %v", fdst)
 	}
 
 	checks := make(chan []Object, Config.Transfers)
@@ -441,7 +384,7 @@ func Check(fdst, fsrc Fs) error {
 				if src.Size() != dst.Size() {
 					Stats.DoneChecking(src)
 					Stats.Error()
-					ErrorLog(src, "Sizes differ")
+					Log(src, "Sizes differ")
 					continue
 				}
 				same, err := CheckMd5sums(src, dst)
@@ -451,7 +394,7 @@ func Check(fdst, fsrc Fs) error {
 				}
 				if !same {
 					Stats.Error()
-					ErrorLog(src, "Md5sums differ")
+					Log(src, "Md5sums differ")
 				}
 				Debug(src, "OK")
 			}
@@ -486,24 +429,14 @@ func ListFn(f Fs, fn func(Object)) error {
 	return nil
 }
 
-// mutex for synchronized output
-var outMutex sync.Mutex
-
-// Synchronized fmt.Fprintf
-func syncFprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
-	outMutex.Lock()
-	defer outMutex.Unlock()
-	return fmt.Fprintf(w, format, a...)
-}
-
 // List the Fs to stdout
 //
 // Shows size and path
 //
 // Lists in parallel which may get them out of order
-func List(f Fs, w io.Writer) error {
+func List(f Fs) error {
 	return ListFn(f, func(o Object) {
-		syncFprintf(w, "%9d %s\n", o.Size(), o.Remote())
+		fmt.Printf("%9d %s\n", o.Size(), o.Remote())
 	})
 }
 
@@ -512,12 +445,12 @@ func List(f Fs, w io.Writer) error {
 // Shows size, mod time and path
 //
 // Lists in parallel which may get them out of order
-func ListLong(f Fs, w io.Writer) error {
+func ListLong(f Fs) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		modTime := o.ModTime()
 		Stats.DoneChecking(o)
-		syncFprintf(w, "%9d %s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.000000000"), o.Remote())
+		fmt.Printf("%9d %19s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.00000000"), o.Remote())
 	})
 }
 
@@ -526,23 +459,23 @@ func ListLong(f Fs, w io.Writer) error {
 // Produces the same output as the md5sum command
 //
 // Lists in parallel which may get them out of order
-func Md5sum(f Fs, w io.Writer) error {
+func Md5sum(f Fs) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		md5sum, err := o.Md5sum()
 		Stats.DoneChecking(o)
 		if err != nil {
 			Debug(o, "Failed to read MD5: %v", err)
-			md5sum = "ERROR"
+			md5sum = "UNKNOWN"
 		}
-		syncFprintf(w, "%32s  %s\n", md5sum, o.Remote())
+		fmt.Printf("%32s  %s\n", md5sum, o.Remote())
 	})
 }
 
 // List the directories/buckets/containers in the Fs to stdout
-func ListDir(f Fs, w io.Writer) error {
+func ListDir(f Fs) error {
 	for dir := range f.ListDir() {
-		syncFprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
+		fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
 	}
 	return nil
 }
@@ -575,21 +508,20 @@ func Rmdir(f Fs) error {
 //
 // FIXME doesn't delete local directories
 func Purge(f Fs) error {
-	var err error
 	if purger, ok := f.(Purger); ok {
 		if Config.DryRun {
 			Debug(f, "Not purging as --dry-run set")
 		} else {
-			err = purger.Purge()
+			err := purger.Purge()
+			if err != nil {
+				Stats.Error()
+				return err
+			}
 		}
 	} else {
-		// DeleteFiles and Rmdir observe --dry-run
 		DeleteFiles(f.List())
-		err = Rmdir(f)
-	}
-	if err != nil {
-		Stats.Error()
-		return err
+		log.Printf("Deleting path")
+		Rmdir(f)
 	}
 	return nil
 }

fs/operations_test.go

@@ -1,468 +0,0 @@
-// Test rclone by doing real transactions to a storage provider to and
-// from the local disk
-
-package fs_test
-
-import (
-	"bytes"
-	"flag"
-	"io/ioutil"
-	"log"
-	"os"
-	"path"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest"
-
-	// Active file systems
-	_ "github.com/ncw/rclone/drive"
-	_ "github.com/ncw/rclone/dropbox"
-	_ "github.com/ncw/rclone/googlecloudstorage"
-	_ "github.com/ncw/rclone/local"
-	_ "github.com/ncw/rclone/s3"
-	_ "github.com/ncw/rclone/swift"
-)
-
-// Globals
-var (
-	localName, remoteName string
-	flocal, fremote       fs.Fs
-	RemoteName            = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
-	SubDir                = flag.Bool("subdir", false, "Set to test with a sub directory")
-	Verbose               = flag.Bool("verbose", false, "Set to enable logging")
-	finalise              func()
-)
-
-// Write a file
-func WriteFile(filePath, content string, t time.Time) {
-	// FIXME make directories?
-	filePath = path.Join(localName, filePath)
-	dirPath := path.Dir(filePath)
-	err := os.MkdirAll(dirPath, 0770)
-	if err != nil {
-		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
-	}
-	err = ioutil.WriteFile(filePath, []byte(content), 0600)
-	if err != nil {
-		log.Fatalf("Failed to write file %q: %v", filePath, err)
-	}
-	err = os.Chtimes(filePath, t, t)
-	if err != nil {
-		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
-	}
-}
-
-var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
-var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
-var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
-
-func TestInit(t *testing.T) {
-	fs.LoadConfig()
-	fs.Config.Verbose = *Verbose
-	fs.Config.Quiet = !*Verbose
-	var err error
-	fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
-	if err != nil {
-		t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
-	}
-	t.Logf("Testing with remote %v", fremote)
-
-	localName, err = ioutil.TempDir("", "rclone")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	localName = filepath.ToSlash(localName)
-	t.Logf("Testing with local %q", localName)
-	flocal, err = fs.NewFs(localName)
-	if err != nil {
-		t.Fatalf("Failed to make %q: %v", remoteName, err)
-	}
-
-}
-func TestCalculateModifyWindow(t *testing.T) {
-	fs.CalculateModifyWindow(fremote, flocal)
-	t.Logf("ModifyWindow is %q", fs.Config.ModifyWindow)
-}
-
-func TestMkdir(t *testing.T) {
-	fstest.TestMkdir(t, fremote)
-}
-
-// Check dry run is working
-func TestCopyWithDryRun(t *testing.T) {
-	WriteFile("sub dir/hello world", "hello world", t1)
-
-	fs.Config.DryRun = true
-	err := fs.Sync(fremote, flocal, false)
-	fs.Config.DryRun = false
-	if err != nil {
-		t.Fatalf("Copy failed: %v", err)
-	}
-
-	items := []fstest.Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, []fstest.Item{}, fs.Config.ModifyWindow)
-}
-
-// Now without dry run
-func TestCopy(t *testing.T) {
-	err := fs.Sync(fremote, flocal, false)
-	if err != nil {
-		t.Fatalf("Copy failed: %v", err)
-	}
-
-	items := []fstest.Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-func TestLsd(t *testing.T) {
-	var buf bytes.Buffer
-	err := fs.ListDir(fremote, &buf)
-	if err != nil {
-		t.Fatalf("ListDir failed: %v", err)
-	}
-	res := buf.String()
-	if !strings.Contains(res, "sub dir\n") {
-		t.Fatalf("Result wrong %q", res)
-	}
-}
-
-// Now delete the local file and download it
-func TestCopyAfterDelete(t *testing.T) {
-	err := os.Remove(localName + "/sub dir/hello world")
-	if err != nil {
-		t.Fatalf("Remove failed: %v", err)
-	}
-
-	items := []fstest.Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, []fstest.Item{}, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-func TestCopyRedownload(t *testing.T) {
-	err := fs.Sync(flocal, fremote, false)
-	if err != nil {
-		t.Fatalf("Copy failed: %v", err)
-	}
-
-	items := []fstest.Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-
-	// Clean the directory
-	cleanTempDir(t)
-}
-
-// Create a file and sync it. Change the last modified date and resync.
-// If we're only doing sync by size and checksum, we expect nothing to
-// to be transferred on the second sync.
-func TestSyncBasedOnCheckSum(t *testing.T) {
-	cleanTempDir(t)
-	fs.Config.CheckSum = true
-	defer func() { fs.Config.CheckSum = false }()
-
-	WriteFile("check sum", "", t1)
-	local_items := []fstest.Item{
-		{Path: "check sum", Size: 0, ModTime: t1, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-
-	fs.Stats.ResetCounters()
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Initial sync failed: %v", err)
-	}
-
-	// We should have transferred exactly one file.
-	if fs.Stats.GetTransfers() != 1 {
-		t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers())
-	}
-
-	remote_items := local_items
-	fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
-
-	err = os.Chtimes(localName+"/check sum", t2, t2)
-	if err != nil {
-		t.Fatalf("Chtimes failed: %v", err)
-	}
-	local_items = []fstest.Item{
-		{Path: "check sum", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-
-	fs.Stats.ResetCounters()
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-
-	// We should have transferred no files
-	if fs.Stats.GetTransfers() != 0 {
-		t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers())
-	}
-
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
-
-	cleanTempDir(t)
-}
-
-// Create a file and sync it. Change the last modified date and the
-// file contents but not the size.  If we're only doing sync by size
-// only, we expect nothing to to be transferred on the second sync.
-func TestSyncSizeOnly(t *testing.T) {
-	cleanTempDir(t)
-	fs.Config.SizeOnly = true
-	defer func() { fs.Config.SizeOnly = false }()
-
-	WriteFile("sizeonly", "potato", t1)
-	local_items := []fstest.Item{
-		{Path: "sizeonly", Size: 6, ModTime: t1, Md5sum: "8ee2027983915ec78acc45027d874316"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-
-	fs.Stats.ResetCounters()
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Initial sync failed: %v", err)
-	}
-
-	// We should have transferred exactly one file.
-	if fs.Stats.GetTransfers() != 1 {
-		t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers())
-	}
-
-	remote_items := local_items
-	fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
-
-	// Update mtime, md5sum but not length of file
-	WriteFile("sizeonly", "POTATO", t2)
-	local_items = []fstest.Item{
-		{Path: "sizeonly", Size: 6, ModTime: t2, Md5sum: "8ac6f27a282e4938125482607ccfb55f"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-
-	fs.Stats.ResetCounters()
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-
-	// We should have transferred no files
-	if fs.Stats.GetTransfers() != 0 {
-		t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers())
-	}
-
-	fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
-
-	cleanTempDir(t)
-}
-
-func TestSyncAfterChangingModtimeOnly(t *testing.T) {
-	WriteFile("empty space", "", t1)
-
-	err := os.Chtimes(localName+"/empty space", t2, t2)
-	if err != nil {
-		t.Fatalf("Chtimes failed: %v", err)
-	}
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-func TestSyncAfterAddingAFile(t *testing.T) {
-	WriteFile("potato", "------------------------------------------------------------", t3)
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
-	WriteFile("potato", "smaller but same date", t3)
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-// Sync after changing a file's contents, modtime but not length
-func TestSyncAfterChangingContentsOnly(t *testing.T) {
-	WriteFile("potato", "SMALLER BUT SAME DATE", t2)
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-// Sync after removing a file and adding a file --dry-run
-func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
-	WriteFile("potato2", "------------------------------------------------------------", t1)
-	err := os.Remove(localName + "/potato")
-	if err != nil {
-		t.Fatalf("Remove failed: %v", err)
-	}
-	fs.Config.DryRun = true
-	err = fs.Sync(fremote, flocal, true)
-	fs.Config.DryRun = false
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-
-	before := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, before, fs.Config.ModifyWindow)
-}
-
-// Sync after removing a file and adding a file
-func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
-	err := fs.Sync(fremote, flocal, true)
-	if err != nil {
-		t.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
-	fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
-}
-
-func TestLs(t *testing.T) {
-	var buf bytes.Buffer
-	err := fs.List(fremote, &buf)
-	if err != nil {
-		t.Fatalf("List failed: %v", err)
-	}
-	res := buf.String()
-	if !strings.Contains(res, "        0 empty space\n") {
-		t.Errorf("empty space missing: %q", res)
-	}
-	if !strings.Contains(res, "       60 potato2\n") {
-		t.Errorf("potato2 missing: %q", res)
-	}
-}
-
-func TestLsLong(t *testing.T) {
-	var buf bytes.Buffer
-	err := fs.ListLong(fremote, &buf)
-	if err != nil {
-		t.Fatalf("List failed: %v", err)
-	}
-	res := buf.String()
-	lines := strings.Split(strings.Trim(res, "\n"), "\n")
-	if len(lines) != 2 {
-		t.Fatalf("Wrong number of lines in list: %q", lines)
-	}
-
-	timeFormat := "2006-01-02 15:04:05.000000000"
-	precision := fremote.Precision()
-	checkTime := func(m, filename string, expected time.Time) {
-		modTime, err := time.Parse(timeFormat, m)
-		if err != nil {
-			t.Errorf("Error parsing %q: %v", m, err)
-		} else {
-			dt, ok := fstest.CheckTimeEqualWithPrecision(expected, modTime, precision)
-			if !ok {
-				t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", filename, dt, precision, modTime, expected, precision)
-			}
-		}
-	}
-
-	m1 := regexp.MustCompile(`(?m)^        0 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) empty space$`)
-	if ms := m1.FindStringSubmatch(res); ms == nil {
-		t.Errorf("empty space missing: %q", res)
-	} else {
-		checkTime(ms[1], "empty space", t2.Local())
-	}
-
-	m2 := regexp.MustCompile(`(?m)^       60 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) potato2$`)
-	if ms := m2.FindStringSubmatch(res); ms == nil {
-		t.Errorf("potato2 missing: %q", res)
-	} else {
-		checkTime(ms[1], "potato2", t1.Local())
-	}
-}
-
-func TestMd5sum(t *testing.T) {
-	var buf bytes.Buffer
-	err := fs.Md5sum(fremote, &buf)
-	if err != nil {
-		t.Fatalf("List failed: %v", err)
-	}
-	res := buf.String()
-	if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e  empty space\n") &&
-		!strings.Contains(res, "                                  empty space\n") {
-		t.Errorf("empty space missing: %q", res)
-	}
-	if !strings.Contains(res, "6548b156ea68a4e003e786df99eee76  potato2\n") &&
-		!strings.Contains(res, "                                  potato2\n") {
-		t.Errorf("potato2 missing: %q", res)
-	}
-}
-
-func TestCheck(t *testing.T) {
-}
-
-// Clean the temporary directory
-func cleanTempDir(t *testing.T) {
-	t.Logf("Cleaning temporary directory: %q", localName)
-	err := os.RemoveAll(localName)
-	if err != nil {
-		t.Logf("Failed to remove %q: %v", localName, err)
-	}
-}
-
-func TestFinalise(t *testing.T) {
-	finalise()
-
-	cleanTempDir(t)
-}

fs/test_all.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-go install
-
-REMOTES="
-TestSwift:
-TestS3:
-TestDrive:
-TestGoogleCloudStorage:
-TestDropbox:
-"
-
-function test_remote {
-    args=$@
-    echo "@go test $args"
-    go test $args || {
-        echo "*** test $args FAILED ***"
-        exit 1
-    }
-}
-
-test_remote
-test_remote --subdir
-for remote in $REMOTES; do
-    test_remote --remote $remote
-    test_remote --remote $remote --subdir
-done
-
-echo "All OK"

fs/version.go

@@ -1,3 +1,3 @@
 package fs
 
-const Version = "v1.18"
+const Version = "v1.02"

fstest/fstest.go

@@ -1,240 +0,0 @@
-// Utilities for testing the Fs
-package fstest
-
-// FIXME put name of test FS in Fs structure
-
-import (
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-)
-
-// Seed the random number generator
-func init() {
-	rand.Seed(time.Now().UnixNano())
-
-}
-
-// Represents an item for checking
-type Item struct {
-	Path    string
-	Md5sum  string
-	ModTime time.Time
-	Size    int64
-}
-
-// Checks the times are equal within the precision, returns the delta and a flag
-func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
-	dt := t0.Sub(t1)
-	if dt >= precision || dt <= -precision {
-		return dt, false
-	}
-	return dt, true
-}
-
-// check the mod time to the given precision
-func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
-	dt, ok := CheckTimeEqualWithPrecision(modTime, i.ModTime, precision)
-	if !ok {
-		t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", obj.Remote(), dt, precision, modTime, i.ModTime, precision)
-	}
-}
-
-func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
-	if obj == nil {
-		t.Fatalf("Object is nil")
-	}
-	// Check attributes
-	Md5sum, err := obj.Md5sum()
-	if err != nil {
-		t.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
-	}
-	if !fs.Md5sumsEqual(i.Md5sum, Md5sum) {
-		t.Errorf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
-	}
-	if i.Size != obj.Size() {
-		t.Errorf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
-	}
-	i.CheckModTime(t, obj, obj.ModTime(), precision)
-}
-
-// Represents all items for checking
-type Items struct {
-	byName map[string]*Item
-	items  []Item
-}
-
-// Make an Items
-func NewItems(items []Item) *Items {
-	is := &Items{
-		byName: make(map[string]*Item),
-		items:  items,
-	}
-	// Fill up byName
-	for i := range items {
-		is.byName[items[i].Path] = &items[i]
-	}
-	return is
-}
-
-// Check off an item
-func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
-	i, ok := is.byName[obj.Remote()]
-	if !ok {
-		t.Errorf("Unexpected file %q", obj.Remote())
-		return
-	}
-	delete(is.byName, obj.Remote())
-	i.Check(t, obj, precision)
-}
-
-// Check all done
-func (is *Items) Done(t *testing.T) {
-	if len(is.byName) != 0 {
-		for name := range is.byName {
-			log.Printf("Not found %q", name)
-		}
-		t.Errorf("%d objects not found", len(is.byName))
-	}
-}
-
-// Checks the fs to see if it has the expected contents
-func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
-	is := NewItems(items)
-	for obj := range f.List() {
-		if obj == nil {
-			t.Errorf("Unexpected nil in List()")
-			continue
-		}
-		is.Find(t, obj, precision)
-	}
-	is.Done(t)
-}
-
-// Checks the fs to see if it has the expected contents
-func CheckListing(t *testing.T, f fs.Fs, items []Item) {
-	precision := f.Precision()
-	CheckListingWithPrecision(t, f, items, precision)
-}
-
-// Parse a time string or explode
-func Time(timeString string) time.Time {
-	t, err := time.Parse(time.RFC3339Nano, timeString)
-	if err != nil {
-		log.Fatalf("Failed to parse time %q: %v", timeString, err)
-	}
-	return t
-}
-
-// Create a random string
-func RandomString(n int) string {
-	source := "abcdefghijklmnopqrstuvwxyz0123456789"
-	out := make([]byte, n)
-	for i := range out {
-		out[i] = source[rand.Intn(len(source))]
-	}
-	return string(out)
-}
-
-// Creates a temporary directory name for local remotes
-func LocalRemote() (path string, err error) {
-	path, err = ioutil.TempDir("", "rclone")
-	if err == nil {
-		// Now remove the directory
-		err = os.Remove(path)
-	}
-	path = filepath.ToSlash(path)
-	return
-}
-
-// Make a random bucket or subdirectory name
-//
-// Returns a random remote name plus the leaf name
-func RandomRemoteName(remoteName string) (string, string, error) {
-	var err error
-	var leafName string
-
-	// Make a directory if remote name is null
-	if remoteName == "" {
-		remoteName, err = LocalRemote()
-		if err != nil {
-			return "", "", err
-		}
-	} else {
-		if !strings.HasSuffix(remoteName, ":") {
-			remoteName += "/"
-		}
-		leafName = RandomString(32)
-		remoteName += leafName
-	}
-	return remoteName, leafName, nil
-}
-
-// Make a random bucket or subdirectory on the remote
-//
-// Call the finalise function returned to Purge the fs at the end (and
-// the parent if necessary)
-func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
-	var err error
-	var parentRemote fs.Fs
-
-	remoteName, _, err = RandomRemoteName(remoteName)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if subdir {
-		parentRemote, err = fs.NewFs(remoteName)
-		if err != nil {
-			return nil, nil, err
-		}
-		remoteName += "/" + RandomString(8)
-	}
-
-	remote, err := fs.NewFs(remoteName)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	finalise := func() {
-		_ = fs.Purge(remote) // ignore error
-		if parentRemote != nil {
-			err = fs.Purge(parentRemote) // ignore error
-			if err != nil {
-				log.Printf("Failed to purge %v: %v", parentRemote, err)
-			}
-		}
-	}
-
-	return remote, finalise, nil
-}
-
-func TestMkdir(t *testing.T, remote fs.Fs) {
-	err := fs.Mkdir(remote)
-	if err != nil {
-		t.Fatalf("Mkdir failed: %v", err)
-	}
-	CheckListing(t, remote, []Item{})
-}
-
-func TestPurge(t *testing.T, remote fs.Fs) {
-	err := fs.Purge(remote)
-	if err != nil {
-		t.Fatalf("Purge failed: %v", err)
-	}
-	CheckListing(t, remote, []Item{})
-}
-
-func TestRmdir(t *testing.T, remote fs.Fs) {
-	err := fs.Rmdir(remote)
-	if err != nil {
-		t.Fatalf("Rmdir failed: %v", err)
-	}
-}

fstest/fstests/fstests.go

@@ -1,447 +0,0 @@
-// Generic tests for testing the Fs and Object interfaces
-package fstests
-
-import (
-	"bytes"
-	"crypto/md5"
-	"encoding/hex"
-	"flag"
-	"io"
-	"log"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest"
-)
-
-var (
-	remote        fs.Fs
-	RemoteName    = ""
-	subRemoteName = ""
-	subRemoteLeaf = ""
-	NilObject     fs.Object
-	file1         = fstest.Item{
-		ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
-		Path:    "file name.txt",
-	}
-	file2 = fstest.Item{
-		ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
-		Path:    `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`,
-	}
-)
-
-func init() {
-	flag.StringVar(&RemoteName, "remote", "", "Set this to override the default remote name (eg s3:)")
-}
-
-func TestInit(t *testing.T) {
-	var err error
-	fs.LoadConfig()
-	fs.Config.Verbose = false
-	fs.Config.Quiet = true
-	t.Logf("Using remote %q", RemoteName)
-	if RemoteName == "" {
-		RemoteName, err = fstest.LocalRemote()
-		if err != nil {
-			log.Fatalf("Failed to create tmp dir: %v", err)
-		}
-	}
-	subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
-	if err != nil {
-		t.Fatalf("Couldn't make remote name: %v", err)
-	}
-
-	remote, err = fs.NewFs(subRemoteName)
-	if err == fs.NotFoundInConfigFile {
-		log.Printf("Didn't find %q in config file - skipping tests", RemoteName)
-		return
-	}
-	if err != nil {
-		t.Fatalf("Couldn't start FS: %v", err)
-	}
-	fstest.TestMkdir(t, remote)
-}
-
-func skipIfNotOk(t *testing.T) {
-	if remote == nil {
-		t.Skip("FS not configured")
-	}
-}
-
-// String returns a description of the FS
-
-func TestFsString(t *testing.T) {
-	skipIfNotOk(t)
-	str := remote.String()
-	if str == "" {
-		t.Fatal("Bad fs.String()")
-	}
-}
-
-type TestFile struct {
-	ModTime time.Time
-	Path    string
-	Size    int64
-	Md5sum  string
-}
-
-func TestFsRmdirEmpty(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.TestRmdir(t, remote)
-}
-
-func TestFsRmdirNotFound(t *testing.T) {
-	skipIfNotOk(t)
-	err := remote.Rmdir()
-	if err == nil {
-		t.Fatalf("Expecting error on Rmdir non existent")
-	}
-}
-
-func TestFsMkdir(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.TestMkdir(t, remote)
-	fstest.TestMkdir(t, remote)
-}
-
-func TestFsListEmpty(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.CheckListing(t, remote, []fstest.Item{})
-}
-
-func TestFsListDirEmpty(t *testing.T) {
-	skipIfNotOk(t)
-	for obj := range remote.ListDir() {
-		t.Errorf("Found unexpected item %q", obj.Name)
-	}
-}
-
-func TestFsNewFsObjectNotFound(t *testing.T) {
-	skipIfNotOk(t)
-	if remote.NewFsObject("potato") != nil {
-		t.Fatal("Didn't expect to find object")
-	}
-}
-
-func findObject(t *testing.T, Name string) fs.Object {
-	obj := remote.NewFsObject(Name)
-	if obj == nil {
-		t.Fatalf("Object not found: %q", Name)
-	}
-	return obj
-}
-
-func testPut(t *testing.T, file *fstest.Item) {
-	buf := bytes.NewBufferString(fstest.RandomString(100))
-	hash := md5.New()
-	in := io.TeeReader(buf, hash)
-
-	file.Size = int64(buf.Len())
-	obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
-	if err != nil {
-		t.Fatal("Put error", err)
-	}
-	file.Md5sum = hex.EncodeToString(hash.Sum(nil))
-	file.Check(t, obj, remote.Precision())
-	// Re-read the object and check again
-	obj = findObject(t, file.Path)
-	file.Check(t, obj, remote.Precision())
-}
-
-func TestFsPutFile1(t *testing.T) {
-	skipIfNotOk(t)
-	testPut(t, &file1)
-}
-
-func TestFsPutFile2(t *testing.T) {
-	skipIfNotOk(t)
-	testPut(t, &file2)
-}
-
-func TestFsListDirFile2(t *testing.T) {
-	skipIfNotOk(t)
-	found := false
-	for obj := range remote.ListDir() {
-		if obj.Name != `hello? sausage` {
-			t.Errorf("Found unexpected item %q", obj.Name)
-		} else {
-			found = true
-		}
-	}
-	if !found {
-		t.Errorf("Didn't find %q", `hello? sausage`)
-	}
-}
-
-func TestFsListDirRoot(t *testing.T) {
-	skipIfNotOk(t)
-	rootRemote, err := fs.NewFs(RemoteName)
-	if err != nil {
-		t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
-	}
-	found := false
-	for obj := range rootRemote.ListDir() {
-		if obj.Name == subRemoteLeaf {
-			found = true
-		}
-	}
-	if !found {
-		t.Errorf("Didn't find %q", subRemoteLeaf)
-	}
-}
-
-func TestFsListRoot(t *testing.T) {
-	skipIfNotOk(t)
-	rootRemote, err := fs.NewFs(RemoteName)
-	if err != nil {
-		t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
-	}
-	// Should either find file1 and file2 or nothing
-	found1 := false
-	file1 := subRemoteLeaf + "/" + file1.Path
-	found2 := false
-	file2 := subRemoteLeaf + "/" + file2.Path
-	count := 0
-	errors := fs.Stats.GetErrors()
-	for obj := range rootRemote.List() {
-		count++
-		if obj.Remote() == file1 {
-			found1 = true
-		}
-		if obj.Remote() == file2 {
-			found2 = true
-		}
-	}
-	errors -= fs.Stats.GetErrors()
-	if count == 0 {
-		if errors == 0 {
-			t.Error("Expecting error if count==0")
-		}
-		return
-	}
-	if found1 && found2 {
-		if errors != 0 {
-			t.Error("Not expecting error if found")
-		}
-		return
-	}
-	t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", file1, found1, file2, found2, count)
-}
-
-func TestFsListFile1(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
-}
-
-func TestFsNewFsObject(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	file1.Check(t, obj, remote.Precision())
-}
-
-func TestFsListFile1and2(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
-}
-
-func TestFsRmdirFull(t *testing.T) {
-	skipIfNotOk(t)
-	err := remote.Rmdir()
-	if err == nil {
-		t.Fatalf("Expecting error on RMdir on non empty remote")
-	}
-}
-
-func TestFsPrecision(t *testing.T) {
-	skipIfNotOk(t)
-	precision := remote.Precision()
-	if precision == fs.ModTimeNotSupported {
-		return
-	}
-	if precision > time.Second || precision < 0 {
-		t.Fatalf("Precision out of range %v", precision)
-	}
-	// FIXME check expected precision
-}
-
-func TestObjectString(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	s := obj.String()
-	if s != file1.Path {
-		t.Errorf("String() wrong %v != %v", s, file1.Path)
-	}
-	obj = NilObject
-	s = obj.String()
-	if s != "<nil>" {
-		t.Errorf("String() wrong %v != %v", s, "<nil>")
-	}
-}
-
-func TestObjectFs(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	if obj.Fs() != remote {
-		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
-	}
-}
-
-func TestObjectRemote(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	if obj.Remote() != file1.Path {
-		t.Errorf("Remote is wrong %v != %v", obj.Remote(), file1.Path)
-	}
-}
-
-func TestObjectMd5sum(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	Md5sum, err := obj.Md5sum()
-	if err != nil {
-		t.Errorf("Error in Md5sum: %v", err)
-	}
-	if !fs.Md5sumsEqual(Md5sum, file1.Md5sum) {
-		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
-	}
-}
-
-func TestObjectModTime(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
-}
-
-func TestObjectSetModTime(t *testing.T) {
-	skipIfNotOk(t)
-	newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
-	obj := findObject(t, file1.Path)
-	obj.SetModTime(newModTime)
-	file1.ModTime = newModTime
-	file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
-	// And make a new object and read it from there too
-	TestObjectModTime(t)
-}
-
-func TestObjectSize(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	if obj.Size() != file1.Size {
-		t.Errorf("Size is wrong %v != %v", obj.Size(), file1.Size)
-	}
-}
-
-func TestObjectOpen(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	in, err := obj.Open()
-	if err != nil {
-		t.Fatalf("Open() return error: %v", err)
-	}
-	hash := md5.New()
-	n, err := io.Copy(hash, in)
-	if err != nil {
-		t.Fatalf("io.Copy() return error: %v", err)
-	}
-	if n != file1.Size {
-		t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
-	}
-	err = in.Close()
-	if err != nil {
-		t.Fatalf("in.Close() return error: %v", err)
-	}
-	Md5sum := hex.EncodeToString(hash.Sum(nil))
-	if !fs.Md5sumsEqual(Md5sum, file1.Md5sum) {
-		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
-	}
-}
-
-func TestObjectUpdate(t *testing.T) {
-	skipIfNotOk(t)
-	buf := bytes.NewBufferString(fstest.RandomString(200))
-	hash := md5.New()
-	in := io.TeeReader(buf, hash)
-
-	file1.Size = int64(buf.Len())
-	obj := findObject(t, file1.Path)
-	err := obj.Update(in, file1.ModTime, file1.Size)
-	if err != nil {
-		t.Fatal("Update error", err)
-	}
-	file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
-	file1.Check(t, obj, remote.Precision())
-	// Re-read the object and check again
-	obj = findObject(t, file1.Path)
-	file1.Check(t, obj, remote.Precision())
-}
-
-func TestObjectStorable(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	if !obj.Storable() {
-		t.Fatalf("Expecting %v to be storable", obj)
-	}
-}
-
-func TestLimitedFs(t *testing.T) {
-	skipIfNotOk(t)
-	remoteName := subRemoteName + "/" + file2.Path
-	file2Copy := file2
-	file2Copy.Path = "z.txt"
-	fileRemote, err := fs.NewFs(remoteName)
-	if err != nil {
-		t.Fatalf("Failed to make remote %q: %v", remoteName, err)
-	}
-	fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
-	_, ok := fileRemote.(*fs.Limited)
-	if !ok {
-		t.Errorf("%v is not a fs.Limited", fileRemote)
-	}
-}
-
-func TestLimitedFsNotFound(t *testing.T) {
-	skipIfNotOk(t)
-	remoteName := subRemoteName + "/not found.txt"
-	fileRemote, err := fs.NewFs(remoteName)
-	if err != nil {
-		t.Fatalf("Failed to make remote %q: %v", remoteName, err)
-	}
-	fstest.CheckListing(t, fileRemote, []fstest.Item{})
-	_, ok := fileRemote.(*fs.Limited)
-	if ok {
-		t.Errorf("%v is is a fs.Limited", fileRemote)
-	}
-}
-
-func TestObjectRemove(t *testing.T) {
-	skipIfNotOk(t)
-	obj := findObject(t, file1.Path)
-	err := obj.Remove()
-	if err != nil {
-		t.Fatal("Remove error", err)
-	}
-	fstest.CheckListing(t, remote, []fstest.Item{file2})
-}
-
-func TestObjectPurge(t *testing.T) {
-	skipIfNotOk(t)
-	fstest.TestPurge(t, remote)
-	err := fs.Purge(remote)
-	if err == nil {
-		t.Fatal("Expecting error after on second purge")
-	}
-}
-
-func TestFinalise(t *testing.T) {
-	skipIfNotOk(t)
-	if strings.HasPrefix(RemoteName, "/") {
-		// Remove temp directory
-		err := os.Remove(RemoteName)
-		if err != nil {
-			log.Printf("Failed to remove %q: %v\n", RemoteName, err)
-		}
-	}
-}

fstest/fstests/gen_tests.go

@@ -1,143 +0,0 @@
-// +build ignore
-
-// Make the test files from fstests.go
-package main
-
-import (
-	"bufio"
-	"html/template"
-	"log"
-	"os"
-	"os/exec"
-	"regexp"
-	"strings"
-)
-
-// Search fstests.go and return all the test function names
-func findTestFunctions() []string {
-	fns := []string{}
-	matcher := regexp.MustCompile(`^func\s+(Test.*?)\(`)
-
-	in, err := os.Open("fstests.go")
-	if err != nil {
-		log.Fatalf("Couldn't open fstests.go: %v", err)
-	}
-	defer in.Close()
-
-	scanner := bufio.NewScanner(in)
-	for scanner.Scan() {
-		line := scanner.Text()
-		matches := matcher.FindStringSubmatch(line)
-		if len(matches) > 0 {
-			fns = append(fns, matches[1])
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		log.Fatalf("Error scanning file: %v", err)
-	}
-	return fns
-}
-
-// Data to substitute
-type Data struct {
-	Regenerate  string
-	FsName      string
-	UpperFsName string
-	TestName    string
-	ObjectName  string
-	Fns         []string
-}
-
-var testProgram = `
-// Test {{ .UpperFsName }} filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: {{ .Regenerate }}
-package {{ .FsName }}_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-	"github.com/ncw/rclone/{{ .FsName }}"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*{{ .FsName }}.FsObject{{ .ObjectName }})(nil))
-	fstests.RemoteName = "{{ .TestName }}"
-}
-
-// Generic tests for the Fs
-{{ range $fn := .Fns }}func {{ $fn }}(t *testing.T){ fstests.{{ $fn }}(t) }
-{{ end }}
-`
-
-// Generate test file piping it through gofmt
-func generateTestProgram(t *template.Template, fns []string, Fsname string) {
-	fsname := strings.ToLower(Fsname)
-	TestName := "Test" + Fsname + ":"
-	outfile := "../../" + fsname + "/" + fsname + "_test.go"
-	// Find last capitalised group to be object name
-	matcher := regexp.MustCompile(`([A-Z][a-z0-9]+)$`)
-	matches := matcher.FindStringSubmatch(Fsname)
-	if len(matches) == 0 {
-		log.Fatalf("Couldn't find object name in %q", Fsname)
-	}
-	ObjectName := matches[1]
-
-	if fsname == "local" {
-		TestName = ""
-	}
-
-	data := Data{
-		Regenerate:  "go run gen_tests.go or make gen_tests",
-		FsName:      fsname,
-		UpperFsName: Fsname,
-		TestName:    TestName,
-		ObjectName:  ObjectName,
-		Fns:         fns,
-	}
-
-	cmd := exec.Command("gofmt")
-
-	log.Printf("Writing %q", outfile)
-	out, err := os.Create(outfile)
-	if err != nil {
-		log.Fatal(err)
-	}
-	cmd.Stdout = out
-
-	gofmt, err := cmd.StdinPipe()
-	if err != nil {
-		log.Fatal(err)
-	}
-	if err = cmd.Start(); err != nil {
-		log.Fatal(err)
-	}
-	if err = t.Execute(gofmt, data); err != nil {
-		log.Fatal(err)
-	}
-	if err = gofmt.Close(); err != nil {
-		log.Fatal(err)
-	}
-	if err = cmd.Wait(); err != nil {
-		log.Fatal(err)
-	}
-	if err = out.Close(); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func main() {
-	fns := findTestFunctions()
-	t := template.Must(template.New("main").Parse(testProgram))
-	generateTestProgram(t, fns, "Local")
-	generateTestProgram(t, fns, "Swift")
-	generateTestProgram(t, fns, "S3")
-	generateTestProgram(t, fns, "Drive")
-	generateTestProgram(t, fns, "GoogleCloudStorage")
-	generateTestProgram(t, fns, "Dropbox")
-	log.Printf("Done")
-}

googleauth/googleauth.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"net/http"
 
 	"code.google.com/p/goauth2/oauth"
 	"github.com/ncw/rclone/fs"
@@ -81,7 +82,7 @@ func (auth *Auth) newTransport(name string) (*oauth.Transport, error) {
 
 	t := &oauth.Transport{
 		Config:    config,
-		Transport: fs.Config.Transport(),
+		Transport: http.DefaultTransport,
 	}
 
 	return t, nil

googlecloudstorage/googlecloudstorage.go

@@ -17,14 +17,15 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"mime"
 	"net/http"
 	"path"
 	"regexp"
 	"strings"
 	"time"
 
-	"google.golang.org/api/googleapi"
-	"google.golang.org/api/storage/v1"
+	"code.google.com/p/google-api-go-client/googleapi"
+	"code.google.com/p/google-api-go-client/storage/v1"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/googleauth"
@@ -33,8 +34,8 @@ import (
 const (
 	rcloneClientId     = "202264815644.apps.googleusercontent.com"
 	rcloneClientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
-	timeFormatIn       = time.RFC3339
-	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
+	RFC3339In          = time.RFC3339
+	RFC3339Out         = "2006-01-02T15:04:05.000000000Z07:00"
 	metaMtime          = "mtime" // key to store mtime under in metadata
 	listChunks         = 256     // chunk size to read directory listings
 )
@@ -42,7 +43,7 @@ const (
 var (
 	// Description of how to auth for this app
 	storageAuth = &googleauth.Auth{
-		Scope:               storage.DevstorageFullControlScope,
+		Scope:               storage.DevstorageFull_controlScope,
 		DefaultClientId:     rcloneClientId,
 		DefaultClientSecret: rcloneClientSecret,
 	}
@@ -214,7 +215,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.Object {
+func (f *FsStorage) NewFsObjectWithInfo(remote string, info *storage.Object) fs.Object {
 	o := &FsObjectStorage{
 		storage: f,
 		remote:  remote,
@@ -235,7 +236,7 @@ func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.
 //
 // May return nil if an error occurred
 func (f *FsStorage) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -251,26 +252,19 @@ func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) {
 		objects, err := list.Do()
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't read bucket %q: %s", f.bucket, err)
+			fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
 			return
 		}
-		if !directories {
-			for _, object := range objects.Items {
-				if !strings.HasPrefix(object.Name, f.root) {
-					fs.Log(f, "Odd name received %q", object.Name)
-					continue
-				}
-				remote := object.Name[rootLength:]
-				fn(remote, object)
+		for _, object := range objects.Items {
+			if directories && !strings.HasSuffix(object.Name, "/") {
+				continue
 			}
-		} else {
-			var object storage.Object
-			for _, prefix := range objects.Prefixes {
-				if !strings.HasSuffix(prefix, "/") {
-					continue
-				}
-				fn(prefix[:len(prefix)-1], &object)
+			if !strings.HasPrefix(object.Name, f.root) {
+				fs.Log(f, "Odd name received %q", object.Name)
+				continue
 			}
+			remote := object.Name[rootLength:]
+			fn(remote, object)
 		}
 		if objects.NextPageToken == "" {
 			break
@@ -286,13 +280,13 @@ func (f *FsStorage) List() fs.ObjectsChan {
 		// Return no objects at top level list
 		close(out)
 		fs.Stats.Error()
-		fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd")
+		fs.Log(f, "Can't list objects at root - choose a bucket using lsd")
 	} else {
 		// List the objects
 		go func() {
 			defer close(out)
 			f.list(false, func(remote string, object *storage.Object) {
-				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
+				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})
@@ -310,7 +304,7 @@ func (f *FsStorage) ListDir() fs.DirChan {
 			defer close(out)
 			if f.projectNumber == "" {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "Can't list buckets without project number")
+				fs.Log(f, "Can't list buckets without project number")
 				return
 			}
 			listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
@@ -318,7 +312,7 @@ func (f *FsStorage) ListDir() fs.DirChan {
 				buckets, err := listBuckets.Do()
 				if err != nil {
 					fs.Stats.Error()
-					fs.ErrorLog(f, "Couldn't list buckets: %v", err)
+					fs.Log(f, "Couldn't list buckets: %v", err)
 					break
 				} else {
 					for _, bucket := range buckets.Items {
@@ -358,8 +352,8 @@ func (f *FsStorage) ListDir() fs.DirChan {
 // The new object may have been created if an error is returned
 func (f *FsStorage) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
 	// Temporary FsObject under construction
-	o := &FsObjectStorage{storage: f, remote: remote}
-	return o, o.Update(in, modTime, size)
+	fs := &FsObjectStorage{storage: f, remote: remote}
+	return fs, fs.Update(in, modTime, size)
 }
 
 // Mkdir creates the bucket if it doesn't exist
@@ -440,7 +434,7 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
 	// read mtime out of metadata if available
 	mtimeString, ok := info.Metadata[metaMtime]
 	if ok {
-		modTime, err := time.Parse(timeFormatIn, mtimeString)
+		modTime, err := time.Parse(RFC3339In, mtimeString)
 		if err == nil {
 			o.modTime = modTime
 			return
@@ -450,7 +444,7 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
 	}
 
 	// Fallback to the Updated time
-	modTime, err := time.Parse(timeFormatIn, info.Updated)
+	modTime, err := time.Parse(RFC3339In, info.Updated)
 	if err != nil {
 		fs.Log(o, "Bad time decode: %v", err)
 	} else {
@@ -490,7 +484,7 @@ func (o *FsObjectStorage) ModTime() time.Time {
 // Returns metadata for an object
 func metadataFromModTime(modTime time.Time) map[string]string {
 	metadata := make(map[string]string, 1)
-	metadata[metaMtime] = modTime.Format(timeFormatOut)
+	metadata[metaMtime] = modTime.Format(RFC3339Out)
 	return metadata
 }
 
@@ -502,12 +496,11 @@ func (o *FsObjectStorage) SetModTime(modTime time.Time) {
 		Name:     o.storage.root + o.remote,
 		Metadata: metadataFromModTime(modTime),
 	}
-	newObject, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do()
+	_, err := o.storage.svc.Objects.Patch(o.storage.bucket, o.storage.root+o.remote, &object).Do()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
+		fs.Log(o, "Failed to update remote mtime: %s", err)
 	}
-	o.setMetaData(newObject)
 }
 
 // Is this object storable
@@ -537,7 +530,7 @@ func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) {
 		return nil, err
 	}
 	if res.StatusCode != 200 {
-		_ = res.Body.Close() // ignore error
+		res.Body.Close()
 		return nil, fmt.Errorf("Bad response: %d: %s", res.StatusCode, res.Status)
 	}
 	return res.Body, nil
@@ -547,21 +540,24 @@ func (o *FsObjectStorage) Open() (in io.ReadCloser, err error) {
 //
 // The new object may have been created if an error is returned
 func (o *FsObjectStorage) Update(in io.Reader, modTime time.Time, size int64) error {
+	// Guess the content type
+	contentType := mime.TypeByExtension(path.Ext(o.remote))
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+
 	object := storage.Object{
 		Bucket:      o.storage.bucket,
 		Name:        o.storage.root + o.remote,
-		ContentType: fs.MimeType(o),
+		ContentType: contentType,
 		Size:        uint64(size),
-		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
+		Updated:     modTime.Format(RFC3339Out), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
 	newObject, err := o.storage.svc.Objects.Insert(o.storage.bucket, &object).Media(in).Name(object.Name).PredefinedAcl(o.storage.objectAcl).Do()
-	if err != nil {
-		return err
-	}
 	// Set the metadata for the new object while we have it
 	o.setMetaData(newObject)
-	return nil
+	return err
 }
 
 // Remove an object

googlecloudstorage/googlecloudstorage_test.go

@@ -1,53 +0,0 @@
-// Test GoogleCloudStorage filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package googlecloudstorage_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-	"github.com/ncw/rclone/googlecloudstorage"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*googlecloudstorage.FsObjectStorage)(nil))
-	fstests.RemoteName = "TestGoogleCloudStorage:"
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

local/local.go

@@ -1,11 +1,6 @@
 // Local filesystem interface
 package local
 
-// Note that all rclone paths should be / separated.  Anything coming
-// from the filepath module will have \ separators on windows so
-// should be converted using filepath.ToSlash.  Windows is quite happy
-// with / separators so there is no need to convert them back.
-
 import (
 	"crypto/md5"
 	"encoding/hex"
@@ -13,12 +8,12 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"github.com/ncw/rclone/fs"
 )
@@ -33,15 +28,14 @@ func init() {
 
 // FsLocal represents a local filesystem rooted at root
 type FsLocal struct {
-	root        string              // The root directory
-	precisionOk sync.Once           // Whether we need to read the precision
-	precision   time.Duration       // precision of local filesystem
-	warned      map[string]struct{} // whether we have warned about this string
+	root        string        // The root directory
+	precisionOk sync.Once     // Whether we need to read the precision
+	precision   time.Duration // precision of local filesystem
 }
 
 // FsObjectLocal represents a local filesystem object
 type FsObjectLocal struct {
-	local  *FsLocal    // The Fs this object is part of
+	local  fs.Fs       // The Fs this object is part of
 	remote string      // The remote path
 	path   string      // The local path
 	info   os.FileInfo // Interface for file info (always present)
@@ -52,11 +46,8 @@ type FsObjectLocal struct {
 
 // NewFs contstructs an FsLocal from the path
 func NewFs(name, root string) (fs.Fs, error) {
-	root = filepath.ToSlash(path.Clean(root))
-	f := &FsLocal{
-		root:   root,
-		warned: make(map[string]struct{}),
-	}
+	root = path.Clean(root)
+	f := &FsLocal{root: root}
 	// Check to see if this points to a file
 	fi, err := os.Lstat(f.root)
 	if err == nil && fi.Mode().IsRegular() {
@@ -78,9 +69,8 @@ func (f *FsLocal) String() string {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
-	remote = filepath.ToSlash(remote)
-	path := path.Join(f.root, remote)
+func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
+	path := filepath.Join(f.root, remote)
 	o := &FsObjectLocal{local: f, remote: remote, path: path}
 	if info != nil {
 		o.info = info
@@ -98,7 +88,7 @@ func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object
 //
 // May return nil if an error occurred
 func (f *FsLocal) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
 // List the path returning a channel of FsObjects
@@ -110,19 +100,19 @@ func (f *FsLocal) List() fs.ObjectsChan {
 		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
 			if err != nil {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err)
+				log.Printf("Failed to open directory: %s: %s", path, err)
 			} else {
 				remote, err := filepath.Rel(f.root, path)
 				if err != nil {
 					fs.Stats.Error()
-					fs.ErrorLog(f, "Failed to get relative path %s: %s", path, err)
+					log.Printf("Failed to get relative path %s: %s", path, err)
 					return nil
 				}
 				if remote == "." {
 					return nil
 					// remote = ""
 				}
-				if fs := f.newFsObjectWithInfo(remote, fi); fs != nil {
+				if fs := f.NewFsObjectWithInfo(remote, fi); fs != nil {
 					if fs.Storable() {
 						out <- fs
 					}
@@ -132,27 +122,13 @@ func (f *FsLocal) List() fs.ObjectsChan {
 		})
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Failed to open directory: %s: %s", f.root, err)
+			log.Printf("Failed to open directory: %s: %s", f.root, err)
 		}
 		close(out)
 	}()
 	return out
 }
 
-// CleanUtf8 makes string a valid UTF-8 string
-//
-// Any invalid UTF-8 characters will be replaced with utf8.RuneError
-func (f *FsLocal) cleanUtf8(name string) string {
-	if utf8.ValidString(name) {
-		return name
-	}
-	if _, ok := f.warned[name]; !ok {
-		fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
-		f.warned[name] = struct{}{}
-	}
-	return string([]rune(name))
-}
-
 // Walk the path returning a channel of FsObjects
 func (f *FsLocal) ListDir() fs.DirChan {
 	out := make(fs.DirChan, fs.Config.Checkers)
@@ -161,12 +137,12 @@ func (f *FsLocal) ListDir() fs.DirChan {
 		items, err := ioutil.ReadDir(f.root)
 		if err != nil {
 			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't find read directory: %s", err)
+			log.Printf("Couldn't find read directory: %s", err)
 		} else {
 			for _, item := range items {
 				if item.IsDir() {
 					dir := &fs.Dir{
-						Name:  f.cleanUtf8(item.Name()),
+						Name:  item.Name(),
 						When:  item.ModTime(),
 						Bytes: 0,
 						Count: 0,
@@ -176,7 +152,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
 						if err != nil {
 							fs.Stats.Error()
-							fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err)
+							log.Printf("Failed to open directory: %s: %s", path, err)
 						} else {
 							dir.Count += 1
 							dir.Bytes += fi.Size()
@@ -185,7 +161,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 					})
 					if err != nil {
 						fs.Stats.Error()
-						fs.ErrorLog(f, "Failed to open directory: %s: %s", dirpath, err)
+						log.Printf("Failed to open directory: %s: %s", dirpath, err)
 					}
 					out <- dir
 				}
@@ -198,7 +174,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 
 // Puts the FsObject to the local filesystem
 func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
-	dstPath := path.Join(f.root, remote)
+	dstPath := filepath.Join(f.root, remote)
 	// Temporary FsObject under construction - info filled in by Update()
 	o := &FsObjectLocal{local: f, remote: remote, path: dstPath}
 	err := o.Update(in, modTime, size)
@@ -210,7 +186,7 @@ func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64
 
 // Mkdir creates the directory if it doesn't exist
 func (f *FsLocal) Mkdir() error {
-	return os.MkdirAll(f.root, 0777)
+	return os.MkdirAll(f.root, 0770)
 }
 
 // Rmdir removes the directory
@@ -242,15 +218,12 @@ func (f *FsLocal) readPrecision() (precision time.Duration) {
 	}
 	path := fd.Name()
 	// fmt.Println("Created temp file", path)
-	err = fd.Close()
-	if err != nil {
-		return time.Second
-	}
+	fd.Close()
 
 	// Delete it on return
 	defer func() {
 		// fmt.Println("Remove temp file")
-		_ = os.Remove(path) // ignore error
+		os.Remove(path)
 	}()
 
 	// Find the minimum duration we can detect
@@ -286,13 +259,6 @@ func (f *FsLocal) readPrecision() (precision time.Duration) {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *FsLocal) Purge() error {
-	fi, err := os.Lstat(f.root)
-	if err != nil {
-		return err
-	}
-	if !fi.Mode().IsDir() {
-		return fmt.Errorf("Can't Purge non directory: %q", f.root)
-	}
 	return os.RemoveAll(f.root)
 }
 
@@ -313,7 +279,7 @@ func (o *FsObjectLocal) String() string {
 
 // Return the remote path
 func (o *FsObjectLocal) Remote() string {
-	return o.local.cleanUtf8(o.remote)
+	return o.remote
 }
 
 // Md5sum calculates the Md5sum of a file returning a lowercase hex string
@@ -324,7 +290,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
 	in, err := os.Open(o.path)
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to open: %s", err)
+		fs.Log(o, "Failed to open: %s", err)
 		return "", err
 	}
 	hash := md5.New()
@@ -332,12 +298,12 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
 	closeErr := in.Close()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to read: %s", err)
+		fs.Log(o, "Failed to read: %s", err)
 		return "", err
 	}
 	if closeErr != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to close: %s", closeErr)
+		fs.Log(o, "Failed to close: %s", closeErr)
 		return "", closeErr
 	}
 	o.md5sum = hex.EncodeToString(hash.Sum(nil))
@@ -359,13 +325,6 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
 	err := os.Chtimes(o.path, modTime, modTime)
 	if err != nil {
 		fs.Debug(o, "Failed to set mtime on file: %s", err)
-		return
-	}
-	// Re-read metadata
-	err = o.lstat()
-	if err != nil {
-		fs.Debug(o, "Failed to stat: %s", err)
-		return
 	}
 }
 
@@ -376,7 +335,7 @@ func (o *FsObjectLocal) Storable() bool {
 		fs.Debug(o, "Can't transfer non file/directory")
 		return false
 	} else if mode&os.ModeDir != 0 {
-		// fs.Debug(o, "Skipping directory")
+		fs.Debug(o, "FIXME Skipping directory")
 		return false
 	}
 	return true
@@ -429,7 +388,7 @@ func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
 // Update the object from in with modTime and size
 func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
 	dir := path.Dir(o.path)
-	err := os.MkdirAll(dir, 0777)
+	err := os.MkdirAll(dir, 0770)
 	if err != nil {
 		return err
 	}

local/local_test.go

@@ -1,53 +0,0 @@
-// Test Local filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package local_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-	"github.com/ncw/rclone/local"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*local.FsObjectLocal)(nil))
-	fstests.RemoteName = ""
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

make_manual.py

@@ -1,78 +0,0 @@
-#!/usr/bin/python
-"""
-Make single page versions of the documentation for release and
-conversion into man pages etc.
-"""
-
-import os
-import re
-from datetime import datetime
-
-docpath = "docs/content"
-outfile = "MANUAL.md"
-
-# Order to add docs segments to make outfile
-docs = [
-    "about.md",
-    "install.md",
-    "docs.md",
-    "drive.md",
-    "s3.md",
-    "swift.md",
-    "dropbox.md",
-    "googlecloudstorage.md",
-    "local.md",
-    "changelog.md",
-    "bugs.md",
-    "faq.md",
-    "licence.md",
-    "authors.md",
-    "contact.md",
-]
-
-# Docs which aren't made into outfile
-ignore_docs = [
-    "downloads.md",
-]
-
-def read_doc(doc):
-    """Read file as a string"""
-    path = os.path.join(docpath, doc)
-    with open(path) as fd:
-        contents = fd.read()
-    parts = contents.split("---\n", 2)
-    if len(parts) != 3:
-        raise ValueError("Couldn't find --- markers: found %d parts" % len(parts))
-    contents = parts[2].strip()+"\n\n"
-    # Remove icons
-    contents = re.sub(r'<i class="fa.*?</i>\s*', "", contents)
-    # Make [...](/links/) absolute
-    contents = re.sub(r'\((\/.*?\/)\)', r"(http://rclone.org\1)", contents)
-    return contents
-
-def check_docs(docpath):
-    """Check all the docs are in docpath"""
-    files = set(f for f in os.listdir(docpath) if f.endswith(".md"))
-    files -= set(ignore_docs)
-    docs_set = set(docs)
-    if files == docs_set:
-        return
-    print "Files on disk but not in docs variable: %s" % ", ".join(files - docs_set)
-    print "Files in docs variable but not on disk: %s" % ", ".join(docs_set - files)
-    raise ValueError("Missing files")
-
-def main():
-    check_docs(docpath)
-    with open(outfile, "w") as out:
-        out.write("""\
-%% rclone(1) User Manual
-%% Nick Craig-Wood
-%% %s
-
-""" % datetime.now().strftime("%b %d, %Y"))
-        for doc in docs:
-            out.write(read_doc(doc))
-    print "Written '%s'" % outfile
-
-if __name__ == "__main__":
-    main()

notes.txt

@@ -1,26 +1,3 @@
-Perhaps make Md5sum() and Modtime() optional.  Define the zero values
-"" and 0. Make it so we can support remotes which can't do those.
-
-Fix the docs
-  * factor the README.md into the docs directory
-  * create it as part of make by assembling other parts
-  * write long docs about each flag
-
-Change lsd command so it doesn't show -1
-  * Make sure all Fses show -1 for objects Zero for dates etc
-  * Make test?
-
-Put the TestRemote names into the Fs description
-Make test_all.sh use the TestRemote name automatically
-
-Run errcheck and go vet in the make file
-.. Also race detector?
-.. go tool vet -shadow
-
-Get rid of Storable?
-
-Write developer manual
-
 Todo
   * FIXME: More -dry-run checks for object transfer
   * Might be quicker to check md5sums first? for swift <-> swift certainly, and maybe for small files
@@ -28,23 +5,24 @@ Todo
     * if object.PseudoDirectory {
       * fmt.Printf("%9s %19s %s\n", "Directory", "-", fs.Remote())
   * Make Account wrapper
+    * limit bandwidth for a pool of all individual connectinos
+    * do timeouts by setting a limit, seeing whether io has happened
+      and resetting it if it has
     * make Account do progress meter
+  * Make logging controllable with flags (mostly done)
   * -timeout: Make all timeouts be settable with command line parameters
+  * Windows paths? Do we need to translate / and \?
+  * Make a fs.Errorf and count errors and log them at a different level
   * Add max object size to fs metadata - 5GB for swift, infinite for local, ? for s3
     * tie into -max-size flag
   * FIXME Make NewFs to return err.IsAnObject so can put the LimitedFs
     creation in common code? Or try for as much as possible?
   * FIXME Account all the transactons (ls etc) using a different
     Roundtripper wrapper which wraps the transactions?
-
-More rsync features
-  * include
-  * exclude
-  * max size
-  * -c, --checksum              skip based on checksum, not mod-time & size
-
-Ideas for flags
-  * --retries N flag which would make rclone retry a sync until successful or it tried N times.
+  * FIXME write tests for local file system
+  * FIXME implement tests for single file operations in rclonetest
+  * Need to make directory objects otherwise can't upload an empty directory
+    * Or could upload empty directories only?
 
 Ideas
   * could do encryption - put IV into metadata?
@@ -57,6 +35,26 @@ Ideas
   * control times sync (which is slow with some remotes) with -a --archive flag?
   * Copy a glob pattern - could do with LimitedFs
 
+s3
+  * Can maybe set last modified?
+    * https://forums.aws.amazon.com/message.jspa?messageID=214062
+    * Otherwise can set metadata
+  * Returns etag and last modified in bucket list
+
 Bugs
   * Non verbose - not sure number transferred got counted up? CHECK
   * When doing copy it recurses the whole of the destination FS which isn't necessary
+
+Making a release
+  * go install -v ./...
+  * go test ./...
+  * rclonetest/test.sh
+  * make tag
+  * edit README.md
+  * git commit fs/version.go README.md docs/content/downloads.md
+  * make retag
+  * . ~/bin/go-cross
+  * make cross
+  * make upload
+  * make upload_website
+  * git push --tags origin master

rclone.go

@@ -28,9 +28,8 @@ import (
 var (
 	// Flags
 	cpuprofile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
-	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
+	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats")
 	version       = pflag.BoolP("version", "V", false, "Print the version number")
-	logFile       = pflag.StringP("log-file", "", "", "Log everything to this file")
 )
 
 type Command struct {
@@ -59,10 +58,10 @@ func (cmd *Command) checkArgs(args []string) {
 var Commands = []Command{
 	{
 		Name:     "copy",
-		ArgsHelp: "source:path dest:path",
+		ArgsHelp: "source://path dest://path",
 		Help: `
         Copy the source to the destination.  Doesn't transfer
-        unchanged files, testing by size and modification time or
+        unchanged files, testing first by modification time then by
         MD5SUM.  Doesn't delete files from the destination.`,
 		Run: func(fdst, fsrc fs.Fs) {
 			err := fs.Sync(fdst, fsrc, false)
@@ -75,13 +74,13 @@ var Commands = []Command{
 	},
 	{
 		Name:     "sync",
-		ArgsHelp: "source:path dest:path",
+		ArgsHelp: "source://path dest://path",
 		Help: `
-        Sync the source to the destination, changing the destination
-        only.  Doesn't transfer unchanged files, testing by size and
-        modification time or MD5SUM.  Destination is updated to match
-        source, including deleting files if necessary.  Since this can
-        cause data loss, test first with the --dry-run flag.`,
+        Sync the source to the destination.  Doesn't transfer
+        unchanged files, testing first by modification time then by
+        MD5SUM.  Deletes any files that exist in source that don't
+        exist in destination. Since this can cause data loss, test
+        first with the --dry-run flag.`,
 		Run: func(fdst, fsrc fs.Fs) {
 			err := fs.Sync(fdst, fsrc, true)
 			if err != nil {
@@ -93,11 +92,11 @@ var Commands = []Command{
 	},
 	{
 		Name:     "ls",
-		ArgsHelp: "[remote:path]",
+		ArgsHelp: "[remote://path]",
 		Help: `
         List all the objects in the the path with size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.List(fdst, os.Stdout)
+			err := fs.List(fdst)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}
@@ -107,11 +106,11 @@ var Commands = []Command{
 	},
 	{
 		Name:     "lsd",
-		ArgsHelp: "[remote:path]",
+		ArgsHelp: "[remote://path]",
 		Help: `
         List all directories/containers/buckets in the the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListDir(fdst, os.Stdout)
+			err := fs.ListDir(fdst)
 			if err != nil {
 				log.Fatalf("Failed to listdir: %v", err)
 			}
@@ -121,12 +120,11 @@ var Commands = []Command{
 	},
 	{
 		Name:     "lsl",
-		ArgsHelp: "[remote:path]",
+		ArgsHelp: "[remote://path]",
 		Help: `
-        List all the objects in the the path with modification time,
-        size and path.`,
+        List all the objects in the the path with modification time, size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListLong(fdst, os.Stdout)
+			err := fs.ListLong(fdst)
 			if err != nil {
 				log.Fatalf("Failed to list long: %v", err)
 			}
@@ -136,12 +134,11 @@ var Commands = []Command{
 	},
 	{
 		Name:     "md5sum",
-		ArgsHelp: "[remote:path]",
+		ArgsHelp: "[remote://path]",
 		Help: `
-        Produces an md5sum file for all the objects in the path.  This
-        is in the same format as the standard md5sum tool produces.`,
+        Produces an md5sum file for all the objects in the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.Md5sum(fdst, os.Stdout)
+			err := fs.Md5sum(fdst)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}
@@ -151,7 +148,7 @@ var Commands = []Command{
 	},
 	{
 		Name:     "mkdir",
-		ArgsHelp: "remote:path",
+		ArgsHelp: "remote://path",
 		Help: `
         Make the path if it doesn't already exist`,
 		Run: func(fdst, fsrc fs.Fs) {
@@ -165,7 +162,7 @@ var Commands = []Command{
 	},
 	{
 		Name:     "rmdir",
-		ArgsHelp: "remote:path",
+		ArgsHelp: "remote://path",
 		Help: `
         Remove the path.  Note that you can't remove a path with
         objects in it, use purge for that.`,
@@ -180,7 +177,7 @@ var Commands = []Command{
 	},
 	{
 		Name:     "purge",
-		ArgsHelp: "remote:path",
+		ArgsHelp: "remote://path",
 		Help: `
         Remove the path and all of its contents.`,
 		Run: func(fdst, fsrc fs.Fs) {
@@ -194,7 +191,7 @@ var Commands = []Command{
 	},
 	{
 		Name:     "check",
-		ArgsHelp: "source:path dest:path",
+		ArgsHelp: "source://path dest://path",
 		Help: `
         Checks the files in the source and destination match.  It
         compares sizes and MD5SUMs and prints a report of files which
@@ -243,8 +240,7 @@ Subcommands:
 	fmt.Fprintf(os.Stderr, "Options:\n")
 	pflag.PrintDefaults()
 	fmt.Fprintf(os.Stderr, `
-It is only necessary to use a unique prefix of the subcommand, eg 'up'
-for 'upload'.
+It is only necessary to use a unique prefix of the subcommand, eg 'up' for 'upload'.
 `)
 }
 
@@ -269,11 +265,7 @@ func ParseFlags() {
 			fs.Stats.Error()
 			log.Fatal(err)
 		}
-		err = pprof.StartCPUProfile(f)
-		if err != nil {
-			fs.Stats.Error()
-			log.Fatal(err)
-		}
+		pprof.StartCPUProfile(f)
 		defer pprof.StopCPUProfile()
 	}
 }
@@ -327,9 +319,6 @@ func NewFs(remote string) fs.Fs {
 
 // Print the stats every statsInterval
 func StartStats() {
-	if *statsInterval <= 0 {
-		return
-	}
 	go func() {
 		ch := time.Tick(*statsInterval)
 		for {
@@ -347,17 +336,6 @@ func main() {
 	}
 	command, args := ParseCommand()
 
-	// Log file output
-	if *logFile != "" {
-		f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
-		if err != nil {
-			log.Fatalf("Failed to open log file: %v", err)
-		}
-		f.Seek(0, os.SEEK_END)
-		log.SetOutput(f)
-		redirectStderr(f)
-	}
-
 	// Make source and destination fs
 	var fdst, fsrc fs.Fs
 	if len(args) >= 1 {
@@ -377,11 +355,11 @@ func main() {
 	// Run the actual command
 	if command.Run != nil {
 		command.Run(fdst, fsrc)
-		if !command.NoStats && (!fs.Config.Quiet || fs.Stats.Errored() || *statsInterval > 0) {
-			fmt.Fprintln(os.Stderr, fs.Stats)
+		if !command.NoStats {
+			fmt.Println(fs.Stats)
 		}
 		if fs.Config.Verbose {
-			fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
+			log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
 		}
 		if fs.Stats.Errored() {
 			os.Exit(1)

rclonetest/rclonetest.go

@@ -0,0 +1,419 @@
+// Test rclone by doing real transactions to a storage provider to and
+// from the local disk
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ogier/pflag"
+
+	// Active file systems
+	_ "github.com/ncw/rclone/drive"
+	_ "github.com/ncw/rclone/dropbox"
+	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/local"
+	_ "github.com/ncw/rclone/s3"
+	_ "github.com/ncw/rclone/swift"
+)
+
+// Globals
+var (
+	localName, remoteName string
+	version               = pflag.BoolP("version", "V", false, "Print the version number")
+	subDir                = pflag.BoolP("subdir", "S", false, "Test with a sub directory")
+)
+
+// Represents an item for checking
+type Item struct {
+	Path    string
+	Md5sum  string
+	ModTime time.Time
+	Size    int64
+}
+
+// Represents all items for checking
+type Items struct {
+	byName map[string]*Item
+	items  []Item
+}
+
+// Make an Items
+func NewItems(items []Item) *Items {
+	is := &Items{
+		byName: make(map[string]*Item),
+		items:  items,
+	}
+	// Fill up byName
+	for i := range items {
+		is.byName[items[i].Path] = &items[i]
+	}
+	return is
+}
+
+// Check off an item
+func (is *Items) Find(obj fs.Object) {
+	i, ok := is.byName[obj.Remote()]
+	if !ok {
+		log.Fatalf("Unexpected file %q", obj.Remote())
+	}
+	delete(is.byName, obj.Remote())
+	// Check attributes
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
+	}
+	if i.Md5sum != Md5sum {
+		log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
+	}
+	if i.Size != obj.Size() {
+		log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
+	}
+	// check the mod time to the given precision
+	modTime := obj.ModTime()
+	dt := modTime.Sub(i.ModTime)
+	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
+		log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
+	}
+
+}
+
+// Check all done
+func (is *Items) Done() {
+	if len(is.byName) != 0 {
+		for name := range is.byName {
+			log.Printf("Not found %q", name)
+		}
+		log.Fatalf("%d objects not found", len(is.byName))
+	}
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListing(f fs.Fs, items []Item) {
+	is := NewItems(items)
+	for obj := range f.List() {
+		is.Find(obj)
+	}
+	is.Done()
+}
+
+// Parse a time string or explode
+func Time(timeString string) time.Time {
+	t, err := time.Parse(time.RFC3339Nano, timeString)
+	if err != nil {
+		log.Fatalf("Failed to parse time %q: %v", timeString, err)
+	}
+	return t
+}
+
+// Write a file
+func WriteFile(filePath, content string, t time.Time) {
+	// FIXME make directories?
+	filePath = path.Join(localName, filePath)
+	dirPath := path.Dir(filePath)
+	err := os.MkdirAll(dirPath, 0770)
+	if err != nil {
+		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
+	}
+	err = ioutil.WriteFile(filePath, []byte(content), 0600)
+	if err != nil {
+		log.Fatalf("Failed to write file %q: %v", filePath, err)
+	}
+	err = os.Chtimes(filePath, t, t)
+	if err != nil {
+		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
+	}
+}
+
+// Create a random string
+func RandomString(n int) string {
+	source := "abcdefghijklmnopqrstuvwxyz0123456789"
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = source[rand.Intn(len(source))]
+	}
+	return string(out)
+}
+
+func TestMkdir(flocal, fremote fs.Fs) {
+	err := fs.Mkdir(fremote)
+	if err != nil {
+		log.Fatalf("Mkdir failed: %v", err)
+	}
+	items := []Item{}
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+}
+
+var t1 = Time("2001-02-03T04:05:06.499999999Z")
+var t2 = Time("2011-12-25T12:59:59.123456789Z")
+var t3 = Time("2011-12-30T12:59:59.000000000Z")
+
+func TestCopy(flocal, fremote fs.Fs) {
+	WriteFile("sub dir/hello world", "hello world", t1)
+
+	// Check dry run is working
+	log.Printf("Copy with --dry-run")
+	fs.Config.DryRun = true
+	err := fs.Sync(fremote, flocal, false)
+	fs.Config.DryRun = false
+	if err != nil {
+		log.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+
+	CheckListing(flocal, items)
+	CheckListing(fremote, []Item{})
+
+	// Now without dry run
+
+	log.Printf("Copy")
+	err = fs.Sync(fremote, flocal, false)
+	if err != nil {
+		log.Fatalf("Copy failed: %v", err)
+	}
+
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+
+	// Now delete the local file and download it
+
+	err = os.Remove(localName + "/sub dir/hello world")
+	if err != nil {
+		log.Fatalf("Remove failed: %v", err)
+	}
+
+	CheckListing(flocal, []Item{})
+	CheckListing(fremote, items)
+
+	log.Printf("Copy - redownload")
+	err = fs.Sync(flocal, fremote, false)
+	if err != nil {
+		log.Fatalf("Copy failed: %v", err)
+	}
+
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+
+	// Clean the directory
+	cleanTempDir()
+}
+
+func TestSync(flocal, fremote fs.Fs) {
+	WriteFile("empty space", "", t1)
+
+	log.Printf("Sync after changing file modtime only")
+	err := os.Chtimes(localName+"/empty space", t2, t2)
+	if err != nil {
+		log.Fatalf("Chtimes failed: %v", err)
+	}
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
+	}
+	items := []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+
+	// ------------------------------------------------------------
+
+	log.Printf("Sync after adding a file")
+	WriteFile("potato", "------------------------------------------------------------", t3)
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
+	}
+	items = []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+
+	// ------------------------------------------------------------
+
+	log.Printf("Sync after changing a file's size only")
+	WriteFile("potato", "smaller but same date", t3)
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
+	}
+	items = []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+
+	// ------------------------------------------------------------
+
+	log.Printf("Sync after removing a file and adding a file --dry-run")
+	WriteFile("potato2", "------------------------------------------------------------", t1)
+	err = os.Remove(localName + "/potato")
+	if err != nil {
+		log.Fatalf("Remove failed: %v", err)
+	}
+	fs.Config.DryRun = true
+	err = fs.Sync(fremote, flocal, true)
+	fs.Config.DryRun = false
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
+	}
+
+	before := []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+	}
+	items = []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, before)
+
+	log.Printf("Sync after removing a file and adding a file")
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, items)
+}
+
+func TestLs(flocal, fremote fs.Fs) {
+	// Underlying List has been tested above, so we just make sure it runs
+	err := fs.List(fremote)
+	if err != nil {
+		log.Fatalf("List failed: %v", err)
+	}
+}
+
+func TestLsd(flocal, fremote fs.Fs) {
+}
+
+func TestCheck(flocal, fremote fs.Fs) {
+}
+
+func TestPurge(fremote fs.Fs) {
+	err := fs.Purge(fremote)
+	if err != nil {
+		log.Fatalf("Purge failed: %v", err)
+	}
+	unexpected := 0
+	for obj := range fremote.List() {
+		unexpected++
+		log.Printf("Found unexpected item %s", obj.Remote())
+	}
+	if unexpected != 0 {
+		log.Fatalf("exiting as found %d unexpected items", unexpected)
+	}
+}
+
+func TestRmdir(flocal, fremote fs.Fs) {
+	err := fs.Rmdir(fremote)
+	if err != nil {
+		log.Fatalf("Rmdir failed: %v", err)
+	}
+}
+
+func syntaxError() {
+	fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s.
+
+Syntax: [options] remote:
+
+Need a remote: as argument.  This will create a random container or
+directory under it and perform tests on it, deleting it at the end.
+
+Options:
+
+`, fs.Version)
+	pflag.PrintDefaults()
+}
+
+// Clean the temporary directory
+func cleanTempDir() {
+	log.Printf("Cleaning temporary directory: %q", localName)
+	err := os.RemoveAll(localName)
+	if err != nil {
+		log.Printf("Failed to remove %q: %v", localName, err)
+	}
+}
+
+func main() {
+	pflag.Usage = syntaxError
+	pflag.Parse()
+	if *version {
+		fmt.Printf("rclonetest %s\n", fs.Version)
+		os.Exit(0)
+	}
+	fs.LoadConfig()
+	rand.Seed(time.Now().UnixNano())
+	args := pflag.Args()
+
+	if len(args) != 1 {
+		syntaxError()
+		os.Exit(1)
+	}
+
+	remoteName = args[0]
+	if !strings.HasSuffix(remoteName, ":") {
+		remoteName += "/"
+	}
+	remoteName += RandomString(32)
+	var parentRemote fs.Fs
+	if *subDir {
+		var err error
+		parentRemote, err = fs.NewFs(remoteName)
+		if err != nil {
+			log.Fatalf("Failed to make parent %q: %v", remoteName, err)
+		}
+		remoteName += "/" + RandomString(8)
+	}
+	log.Printf("Testing with remote %q", remoteName)
+	var err error
+	localName, err = ioutil.TempDir("", "rclone")
+	if err != nil {
+		log.Fatalf("Failed to create temp dir: %v", err)
+	}
+	log.Printf("Testing with local %q", localName)
+
+	fremote, err := fs.NewFs(remoteName)
+	if err != nil {
+		log.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+	flocal, err := fs.NewFs(localName)
+	if err != nil {
+		log.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+
+	fs.CalculateModifyWindow(fremote, flocal)
+
+	TestMkdir(flocal, fremote)
+	TestCopy(flocal, fremote)
+	TestSync(flocal, fremote)
+	TestLs(flocal, fremote)
+	TestLsd(flocal, fremote)
+	TestCheck(flocal, fremote)
+	TestPurge(fremote)
+	//TestRmdir(flocal, fremote)
+
+	if parentRemote != nil {
+		TestPurge(parentRemote)
+	}
+
+	cleanTempDir()
+	log.Printf("Tests OK")
+}

rclonetest/test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+go install
+
+REMOTES="
+memstore:
+s3:
+drive2:
+gcs:
+dropbox:
+/tmp/z
+"
+
+function test_remote {
+    args=$@
+    rclonetest $args || {
+        echo "*** rclonetest $args FAILED ***"
+        exit 1
+    }
+}
+
+for remote in $REMOTES; do
+    test_remote $remote
+    test_remote --subdir $remote
+done

redirect_stderr.go

@@ -1,15 +0,0 @@
-// Log the panic to the log file - for oses which can't do this
-
-//+build !windows,!unix
-
-package main
-
-import (
-	"log"
-	"os"
-)
-
-// redirectStderr to the file passed in
-func redirectStderr(f *os.File) {
-	log.Printf("Can't redirect stderr to file")
-}

redirect_stderr_unix.go

@@ -1,19 +0,0 @@
-// Log the panic under unix to the log file
-
-//+build unix
-
-package main
-
-import (
-	"log"
-	"os"
-	"syscall"
-)
-
-// redirectStderr to the file passed in
-func redirectStderr(f *os.File) {
-	err := syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
-	if err != nil {
-		log.Fatalf("Failed to redirect stderr to file: %v", err)
-	}
-}

redirect_stderr_windows.go

@@ -1,39 +0,0 @@
-// Log the panic under windows to the log file
-//
-// Code from minix, via
-//
-// http://play.golang.org/p/kLtct7lSUg
-
-//+build windows
-
-package main
-
-import (
-	"log"
-	"os"
-	"syscall"
-)
-
-var (
-	kernel32         = syscall.MustLoadDLL("kernel32.dll")
-	procSetStdHandle = kernel32.MustFindProc("SetStdHandle")
-)
-
-func setStdHandle(stdhandle int32, handle syscall.Handle) error {
-	r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, uintptr(stdhandle), uintptr(handle), 0)
-	if r0 == 0 {
-		if e1 != 0 {
-			return error(e1)
-		}
-		return syscall.EINVAL
-	}
-	return nil
-}
-
-// redirectStderr to the file passed in
-func redirectStderr(f *os.File) {
-	err := setStdHandle(syscall.STD_ERROR_HANDLE, syscall.Handle(f.Fd()))
-	if err != nil {
-		log.Fatalf("Failed to redirect stderr to file: %v", err)
-	}
-}

s3/s3.go

@@ -3,31 +3,20 @@ package s3
 
 // FIXME need to prevent anything but ListDir working for s3://
 
-/*
-Progress of port to aws-sdk
-
- * Don't really need o.meta at all?
-
-What happens if you CTRL-C a multipart upload
-  * get an incomplete upload
-  * disappears when you delete the bucket
-*/
-
 import (
 	"errors"
 	"fmt"
 	"io"
+	"mime"
+	"net/http"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/service"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/ncw/goamz/aws"
+	"github.com/ncw/goamz/s3"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/swift"
 )
@@ -45,48 +34,39 @@ func init() {
 			Name: "secret_access_key",
 			Help: "AWS Secret Access Key (password). ",
 		}, {
-			Name: "region",
-			Help: "Region to connect to.",
+			Name: "endpoint",
+			Help: "Endpoint for S3 API.",
 			Examples: []fs.OptionExample{{
-				Value: "us-east-1",
+				Value: "https://s3.amazonaws.com/",
 				Help:  "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.",
 			}, {
-				Value: "us-west-2",
+				Value: "https://s3-external-1.amazonaws.com",
+				Help:  "US Region, Northern Virginia only.\nLeave location constraint empty.",
+			}, {
+				Value: "https://s3-us-west-2.amazonaws.com",
 				Help:  "US West (Oregon) Region\nNeeds location constraint us-west-2.",
 			}, {
-				Value: "us-west-1",
+				Value: "https://s3-us-west-1.amazonaws.com",
 				Help:  "US West (Northern California) Region\nNeeds location constraint us-west-1.",
 			}, {
-				Value: "eu-west-1",
+				Value: "https://s3-eu-west-1.amazonaws.com",
 				Help:  "EU (Ireland) Region Region\nNeeds location constraint EU or eu-west-1.",
 			}, {
-				Value: "eu-central-1",
-				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
-			}, {
-				Value: "ap-southeast-1",
+				Value: "https://s3-ap-southeast-1.amazonaws.com",
 				Help:  "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.",
 			}, {
-				Value: "ap-southeast-2",
-				Help:  "Asia Pacific (Sydney) Region\nNeeds location constraint ap-southeast-2.",
+				Value: "https://s3-ap-southeast-2.amazonaws.com",
+				Help:  "Asia Pacific (Sydney) Region\nNeeds location constraint .",
 			}, {
-				Value: "ap-northeast-1",
+				Value: "https://s3-ap-northeast-1.amazonaws.com",
 				Help:  "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.",
 			}, {
-				Value: "sa-east-1",
+				Value: "https://s3-sa-east-1.amazonaws.com",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
-			}, {
-				Value: "other-v2-signature",
-				Help:  "If using an S3 clone that only understands v2 signatures - eg Ceph - set this and make sure you set the endpoint.",
-			}, {
-				Value: "other-v4-signature",
-				Help:  "If using an S3 clone that understands v4 signatures set this and make sure you set the endpoint.",
 			}},
-		}, {
-			Name: "endpoint",
-			Help: "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.\nSpecify if using an S3 clone such as Ceph.",
 		}, {
 			Name: "location_constraint",
-			Help: "Location constraint - must be set to match the Region. Used when creating buckets only.",
+			Help: "Location constraint - must be set to match the Endpoint.",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Empty for US Region, Northern Virginia or Pacific Northwest.",
@@ -121,18 +101,16 @@ func init() {
 
 // Constants
 const (
-	metaMtime     = "Mtime" // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	listChunkSize = 1024    // number of items to read at once
-	maxRetries    = 10      // number of retries to make of operations
+	metaMtime = "X-Amz-Meta-Mtime" // the meta key to store mtime in
 )
 
 // FsS3 represents a remote s3 server
 type FsS3 struct {
-	c                  *s3.S3 // the connection to the s3 server
-	bucket             string // the bucket we are working on
-	perm               string // permissions for new buckets / objects
-	root               string // root of the bucket - ignore all objects above this
-	locationConstraint string // location constraint of new buckets
+	c      *s3.S3     // the connection to the s3 server
+	b      *s3.Bucket // the connection to the bucket
+	bucket string     // the bucket we are working on
+	perm   s3.ACL     // permissions for new buckets / objects
+	root   string     // root of the bucket - ignore all objects above this
 }
 
 // FsObjectS3 describes a s3 object
@@ -141,12 +119,12 @@ type FsObjectS3 struct {
 	//
 	// List will read everything but meta - to fill that in need to call
 	// readMetaData
-	s3           *FsS3              // what this object is part of
-	remote       string             // The remote path
-	etag         string             // md5sum of the object
-	bytes        int64              // size of the object
-	lastModified time.Time          // Last modified
-	meta         map[string]*string // The object metadata if known - may be nil
+	s3           *FsS3      // what this object is part of
+	remote       string     // The remote path
+	etag         string     // md5sum of the object
+	bytes        int64      // size of the object
+	lastModified time.Time  // Last modified
+	meta         s3.Headers // The object metadata if known - may be nil
 }
 
 // ------------------------------------------------------------
@@ -185,42 +163,27 @@ func s3Connection(name string) (*s3.S3, error) {
 	if secretAccessKey == "" {
 		return nil, errors.New("secret_access_key not found")
 	}
-	auth := credentials.NewStaticCredentials(accessKeyId, secretAccessKey, "")
+	auth := aws.Auth{AccessKey: accessKeyId, SecretKey: secretAccessKey}
 
-	endpoint := fs.ConfigFile.MustValue(name, "endpoint")
-	region := fs.ConfigFile.MustValue(name, "region")
-	if region == "" && endpoint == "" {
-		endpoint = "https://s3.amazonaws.com/"
+	// FIXME look through all the regions by name and use one of them if found
+
+	// Synthesize the region
+	s3Endpoint := fs.ConfigFile.MustValue(name, "endpoint")
+	if s3Endpoint == "" {
+		s3Endpoint = "https://s3.amazonaws.com/"
 	}
-	if region == "" {
-		region = "us-east-1"
+	region := aws.Region{
+		Name:                 "s3",
+		S3Endpoint:           s3Endpoint,
+		S3LocationConstraint: false,
 	}
-	awsConfig := aws.NewConfig().
-		WithRegion(region).
-		WithMaxRetries(maxRetries).
-		WithCredentials(auth).
-		WithEndpoint(endpoint).
-		WithHTTPClient(fs.Config.Client()).
-		WithS3ForcePathStyle(true)
-	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
-	c := s3.New(awsConfig)
-	if region == "other-v2-signature" {
-		fs.Debug(name, "Using v2 auth")
-		signer := func(req *service.Request) {
-			// Ignore AnonymousCredentials object
-			if req.Service.Config.Credentials == credentials.AnonymousCredentials {
-				return
-			}
-			sign(accessKeyId, secretAccessKey, req.HTTPRequest)
-		}
-		c.Handlers.Sign.Clear()
-		c.Handlers.Sign.PushBack(service.BuildContentLength)
-		c.Handlers.Sign.PushBack(signer)
+	s3LocationConstraint := fs.ConfigFile.MustValue(name, "location_constraint")
+	if s3LocationConstraint != "" {
+		region.Name = s3LocationConstraint
+		region.S3LocationConstraint = true
 	}
-	// Add user agent
-	c.Handlers.Build.PushBack(func(r *service.Request) {
-		r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
-	})
+
+	c := s3.New(auth, region)
 	return c, nil
 }
 
@@ -237,18 +200,14 @@ func NewFs(name, root string) (fs.Fs, error) {
 	f := &FsS3{
 		c:      c,
 		bucket: bucket,
-		// FIXME perm:   s3.Private, // FIXME need user to specify
-		root:               directory,
-		locationConstraint: fs.ConfigFile.MustValue(name, "location_constraint"),
+		b:      c.Bucket(bucket),
+		perm:   s3.Private, // FIXME need user to specify
+		root:   directory,
 	}
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the object exists
-		req := s3.HeadObjectInput{
-			Bucket: &f.bucket,
-			Key:    &directory,
-		}
-		_, err = f.c.HeadObject(&req)
+		_, err = f.b.Head(directory, nil)
 		if err == nil {
 			remote := path.Base(directory)
 			f.root = path.Dir(directory)
@@ -262,28 +221,27 @@ func NewFs(name, root string) (fs.Fs, error) {
 			return fs.NewLimited(f, obj), nil
 		}
 	}
-	// f.listMultipartUploads()
 	return f, nil
 }
 
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object {
+func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
 	o := &FsObjectS3{
 		s3:     f,
 		remote: remote,
 	}
 	if info != nil {
 		// Set info but not meta
-		if info.LastModified == nil {
-			fs.Log(o, "Failed to read last modified")
+		var err error
+		o.lastModified, err = time.Parse(time.RFC3339, info.LastModified)
+		if err != nil {
+			fs.Log(o, "Failed to read last modified: %s", err)
 			o.lastModified = time.Now()
-		} else {
-			o.lastModified = *info.LastModified
 		}
-		o.etag = aws.StringValue(info.ETag)
-		o.bytes = aws.Int64Value(info.Size)
+		o.etag = info.ETag
+		o.bytes = info.Size
 	} else {
 		err := o.readMetaData() // reads info and meta, returning an error
 		if err != nil {
@@ -298,71 +256,42 @@ func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object {
 //
 // May return nil if an error occurred
 func (f *FsS3) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
 //
 // If directories is set it only sends directories
-func (f *FsS3) list(directories bool, fn func(string, *s3.Object)) {
-	maxKeys := int64(listChunkSize)
+func (f *FsS3) list(directories bool, fn func(string, *s3.Key)) {
 	delimiter := ""
 	if directories {
 		delimiter = "/"
 	}
-	var marker *string
-	for {
-		// FIXME need to implement ALL loop
-		req := s3.ListObjectsInput{
-			Bucket:    &f.bucket,
-			Delimiter: &delimiter,
-			Prefix:    &f.root,
-			MaxKeys:   &maxKeys,
-			Marker:    marker,
-		}
-		resp, err := f.c.ListObjects(&req)
-		if err != nil {
-			fs.Stats.Error()
-			fs.ErrorLog(f, "Couldn't read bucket %q: %s", f.bucket, err)
-			break
+	// FIXME need to implement ALL loop
+	objects, err := f.b.List(f.root, delimiter, "", 10000)
+	if err != nil {
+		fs.Stats.Error()
+		fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
+	} else {
+		rootLength := len(f.root)
+		if directories {
+			for _, remote := range objects.CommonPrefixes {
+				if !strings.HasPrefix(remote, f.root) {
+					fs.Log(f, "Odd name received %q", remote)
+					continue
+				}
+				remote := remote[rootLength:]
+				fn(remote, &s3.Key{Key: remote})
+			}
 		} else {
-			rootLength := len(f.root)
-			if directories {
-				for _, commonPrefix := range resp.CommonPrefixes {
-					if commonPrefix.Prefix == nil {
-						fs.Log(f, "Nil common prefix received")
-						continue
-					}
-					remote := *commonPrefix.Prefix
-					if !strings.HasPrefix(remote, f.root) {
-						fs.Log(f, "Odd name received %q", remote)
-						continue
-					}
-					remote = remote[rootLength:]
-					if strings.HasSuffix(remote, "/") {
-						remote = remote[:len(remote)-1]
-					}
-					fn(remote, &s3.Object{Key: &remote})
+			for i := range objects.Contents {
+				object := &objects.Contents[i]
+				if !strings.HasPrefix(object.Key, f.root) {
+					fs.Log(f, "Odd name received %q", object.Key)
+					continue
 				}
-			} else {
-				for _, object := range resp.Contents {
-					key := aws.StringValue(object.Key)
-					if !strings.HasPrefix(key, f.root) {
-						fs.Log(f, "Odd name received %q", key)
-						continue
-					}
-					remote := key[rootLength:]
-					fn(remote, object)
-				}
-			}
-			if !aws.BoolValue(resp.IsTruncated) {
-				break
-			}
-			// Use NextMarker if set, otherwise use last Key
-			if resp.NextMarker == nil || *resp.NextMarker == "" {
-				marker = resp.Contents[len(resp.Contents)-1].Key
-			} else {
-				marker = resp.NextMarker
+				remote := object.Key[rootLength:]
+				fn(remote, object)
 			}
 		}
 	}
@@ -375,12 +304,12 @@ func (f *FsS3) List() fs.ObjectsChan {
 		// Return no objects at top level list
 		close(out)
 		fs.Stats.Error()
-		fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd")
+		fs.Log(f, "Can't list objects at root - choose a bucket using lsd")
 	} else {
 		go func() {
 			defer close(out)
-			f.list(false, func(remote string, object *s3.Object) {
-				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
+			f.list(false, func(remote string, object *s3.Key) {
+				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})
@@ -396,16 +325,15 @@ func (f *FsS3) ListDir() fs.DirChan {
 		// List the buckets
 		go func() {
 			defer close(out)
-			req := s3.ListBucketsInput{}
-			resp, err := f.c.ListBuckets(&req)
+			buckets, err := f.c.ListBuckets()
 			if err != nil {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "Couldn't list buckets: %s", err)
+				fs.Log(f, "Couldn't list buckets: %s", err)
 			} else {
-				for _, bucket := range resp.Buckets {
+				for _, bucket := range buckets {
 					out <- &fs.Dir{
-						Name:  aws.StringValue(bucket.Name),
-						When:  aws.TimeValue(bucket.CreationDate),
+						Name:  bucket.Name,
+						When:  bucket.CreationDate,
 						Bytes: -1,
 						Count: -1,
 					}
@@ -416,14 +344,10 @@ func (f *FsS3) ListDir() fs.DirChan {
 		// List the directories in the path in the bucket
 		go func() {
 			defer close(out)
-			f.list(true, func(remote string, object *s3.Object) {
-				size := int64(0)
-				if object.Size != nil {
-					size = *object.Size
-				}
+			f.list(true, func(remote string, object *s3.Key) {
 				out <- &fs.Dir{
 					Name:  remote,
-					Bytes: size,
+					Bytes: object.Size,
 					Count: 0,
 				}
 			})
@@ -441,18 +365,9 @@ func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (
 
 // Mkdir creates the bucket if it doesn't exist
 func (f *FsS3) Mkdir() error {
-	req := s3.CreateBucketInput{
-		Bucket: &f.bucket,
-		ACL:    &f.perm,
-	}
-	if f.locationConstraint != "" {
-		req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
-			LocationConstraint: &f.locationConstraint,
-		}
-	}
-	_, err := f.c.CreateBucket(&req)
-	if err, ok := err.(awserr.Error); ok {
-		if err.Code() == "BucketAlreadyOwnedByYou" {
+	err := f.b.PutBucket(f.perm)
+	if err, ok := err.(*s3.Error); ok {
+		if err.Code == "BucketAlreadyOwnedByYou" {
 			return nil
 		}
 	}
@@ -463,11 +378,7 @@ func (f *FsS3) Mkdir() error {
 //
 // Returns an error if it isn't empty
 func (f *FsS3) Rmdir() error {
-	req := s3.DeleteBucketInput{
-		Bucket: &f.bucket,
-	}
-	_, err := f.c.DeleteBucket(&req)
-	return err
+	return f.b.DelBucket()
 }
 
 // Return the precision
@@ -495,17 +406,9 @@ func (o *FsObjectS3) Remote() string {
 	return o.remote
 }
 
-var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`)
-
 // Md5sum returns the Md5sum of an object returning a lowercase hex string
 func (o *FsObjectS3) Md5sum() (string, error) {
-	etag := strings.Trim(strings.ToLower(o.etag), `"`)
-	// Check the etag is a valid md5sum
-	if !matchMd5.MatchString(etag) {
-		// fs.Debug(o, "Invalid md5sum (probably multipart uploaded) - ignoring: %q", etag)
-		return "", nil
-	}
-	return etag, nil
+	return strings.Trim(strings.ToLower(o.etag), `"`), nil
 }
 
 // Size returns the size of an object in bytes
@@ -520,12 +423,8 @@ func (o *FsObjectS3) readMetaData() (err error) {
 	if o.meta != nil {
 		return nil
 	}
-	key := o.s3.root + o.remote
-	req := s3.HeadObjectInput{
-		Bucket: &o.s3.bucket,
-		Key:    &key,
-	}
-	resp, err := o.s3.c.HeadObject(&req)
+
+	headers, err := o.s3.b.Head(o.s3.root+o.remote, nil)
 	if err != nil {
 		fs.Debug(o, "Failed to read info: %s", err)
 		return err
@@ -533,17 +432,19 @@ func (o *FsObjectS3) readMetaData() (err error) {
 	var size int64
 	// Ignore missing Content-Length assuming it is 0
 	// Some versions of ceph do this due their apache proxies
-	if resp.ContentLength != nil {
-		size = *resp.ContentLength
+	if contentLength, ok := headers["Content-Length"]; ok {
+		size, err = strconv.ParseInt(contentLength, 10, 64)
+		if err != nil {
+			fs.Debug(o, "Failed to read size from: %q", headers)
+			return err
+		}
 	}
-	o.etag = aws.StringValue(resp.ETag)
+	o.etag = headers["Etag"]
 	o.bytes = size
-	o.meta = resp.Metadata
-	if resp.LastModified == nil {
+	o.meta = headers
+	if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil {
 		fs.Log(o, "Failed to read last modified from HEAD: %s", err)
 		o.lastModified = time.Now()
-	} else {
-		o.lastModified = *resp.LastModified
 	}
 	return nil
 }
@@ -560,11 +461,11 @@ func (o *FsObjectS3) ModTime() time.Time {
 	}
 	// read mtime out of metadata if available
 	d, ok := o.meta[metaMtime]
-	if !ok || d == nil {
+	if !ok {
 		// fs.Debug(o, "No metadata")
 		return o.lastModified
 	}
-	modTime, err := swift.FloatStringToTime(*d)
+	modTime, err := swift.FloatStringToTime(d)
 	if err != nil {
 		fs.Log(o, "Failed to read mtime from object: %s", err)
 		return o.lastModified
@@ -577,27 +478,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
 	err := o.readMetaData()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to read metadata: %s", err)
+		fs.Log(o, "Failed to read metadata: %s", err)
 		return
 	}
-	o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime))
-
-	// Copy the object to itself to update the metadata
-	key := o.s3.root + o.remote
-	sourceKey := o.s3.bucket + "/" + key
-	directive := s3.MetadataDirectiveReplace // replace metadata with that passed in
-	req := s3.CopyObjectInput{
-		Bucket:            &o.s3.bucket,
-		ACL:               &o.s3.perm,
-		Key:               &key,
-		CopySource:        &sourceKey,
-		Metadata:          o.meta,
-		MetadataDirective: &directive,
-	}
-	_, err = o.s3.c.CopyObject(&req)
+	o.meta[metaMtime] = swift.TimeToFloatString(modTime)
+	_, err = o.s3.b.Update(o.s3.root+o.remote, o.s3.perm, o.meta)
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
+		fs.Log(o, "Failed to update remote mtime: %s", err)
 	}
 }
 
@@ -608,66 +496,35 @@ func (o *FsObjectS3) Storable() bool {
 
 // Open an object for read
 func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
-	key := o.s3.root + o.remote
-	req := s3.GetObjectInput{
-		Bucket: &o.s3.bucket,
-		Key:    &key,
-	}
-	resp, err := o.s3.c.GetObject(&req)
-	if err != nil {
-		return nil, err
-	}
-	return resp.Body, nil
+	in, err = o.s3.b.GetReader(o.s3.root + o.remote)
+	return
 }
 
 // Update the Object from in with modTime and size
 func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
-	opts := s3manager.UploadOptions{
-		// PartSize:          64 * 1024 * 1024, use default
-		Concurrency:       2, // limit concurrency
-		LeavePartsOnError: false,
-		S3:                o.s3.c,
-	}
-	uploader := s3manager.NewUploader(&opts)
-
-	// Set the mtime in the meta data
-	metadata := map[string]*string{
-		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
+	// Set the mtime in the headers
+	headers := s3.Headers{
+		metaMtime: swift.TimeToFloatString(modTime),
 	}
 
 	// Guess the content type
-	contentType := fs.MimeType(o)
-
-	key := o.s3.root + o.remote
-	req := s3manager.UploadInput{
-		Bucket:      &o.s3.bucket,
-		ACL:         &o.s3.perm,
-		Key:         &key,
-		Body:        in,
-		ContentType: &contentType,
-		Metadata:    metadata,
-		//ContentLength: &size,
+	contentType := mime.TypeByExtension(path.Ext(o.remote))
+	if contentType == "" {
+		contentType = "application/octet-stream"
 	}
-	_, err := uploader.Upload(&req)
+
+	_, err := o.s3.b.PutReaderHeaders(o.s3.root+o.remote, in, size, contentType, o.s3.perm, headers)
 	if err != nil {
 		return err
 	}
-
 	// Read the metadata from the newly created object
-	o.meta = nil // wipe old metadata
 	err = o.readMetaData()
 	return err
 }
 
 // Remove an object
 func (o *FsObjectS3) Remove() error {
-	key := o.s3.root + o.remote
-	req := s3.DeleteObjectInput{
-		Bucket: &o.s3.bucket,
-		Key:    &key,
-	}
-	_, err := o.s3.c.DeleteObject(&req)
-	return err
+	return o.s3.b.Del(o.s3.root + o.remote)
 }
 
 // Check the interfaces are satisfied

s3/s3_test.go

@@ -1,53 +0,0 @@
-// Test S3 filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package s3_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-	"github.com/ncw/rclone/s3"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*s3.FsObjectS3)(nil))
-	fstests.RemoteName = "TestS3:"
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }

s3/v2sign.go

@@ -1,115 +0,0 @@
-// v2 signing
-
-package s3
-
-import (
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
-	"net/http"
-	"sort"
-	"strings"
-	"time"
-)
-
-// URL parameters that need to be added to the signature
-var s3ParamsToSign = map[string]struct{}{
-	"acl":                          struct{}{},
-	"location":                     struct{}{},
-	"logging":                      struct{}{},
-	"notification":                 struct{}{},
-	"partNumber":                   struct{}{},
-	"policy":                       struct{}{},
-	"requestPayment":               struct{}{},
-	"torrent":                      struct{}{},
-	"uploadId":                     struct{}{},
-	"uploads":                      struct{}{},
-	"versionId":                    struct{}{},
-	"versioning":                   struct{}{},
-	"versions":                     struct{}{},
-	"response-content-type":        struct{}{},
-	"response-content-language":    struct{}{},
-	"response-expires":             struct{}{},
-	"response-cache-control":       struct{}{},
-	"response-content-disposition": struct{}{},
-	"response-content-encoding":    struct{}{},
-}
-
-// sign signs requests using v2 auth
-//
-// Cobbled together from goamz and aws-sdk-go
-func sign(AccessKey, SecretKey string, req *http.Request) {
-	// Set date
-	date := time.Now().UTC().Format(time.RFC1123)
-	req.Header.Set("Date", date)
-
-	// Sort out URI
-	uri := req.URL.Opaque
-	if uri != "" {
-		if strings.HasPrefix(uri, "//") {
-			// Strip off //host/uri
-			uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
-			req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused
-		}
-	} else {
-		uri = req.URL.Path
-	}
-	if uri == "" {
-		uri = "/"
-	}
-
-	// Look through headers of interest
-	var md5 string
-	var contentType string
-	var headersToSign []string
-	for k, v := range req.Header {
-		k = strings.ToLower(k)
-		switch k {
-		case "content-md5":
-			md5 = v[0]
-		case "content-type":
-			contentType = v[0]
-		default:
-			if strings.HasPrefix(k, "x-amz-") {
-				vall := strings.Join(v, ",")
-				headersToSign = append(headersToSign, k+":"+vall)
-			}
-		}
-	}
-	// Make headers of interest into canonical string
-	var joinedHeadersToSign string
-	if len(headersToSign) > 0 {
-		sort.StringSlice(headersToSign).Sort()
-		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
-	}
-
-	// Look for query parameters which need to be added to the signature
-	params := req.URL.Query()
-	var queriesToSign []string
-	for k, vs := range params {
-		if _, ok := s3ParamsToSign[k]; ok {
-			for _, v := range vs {
-				if v == "" {
-					queriesToSign = append(queriesToSign, k)
-				} else {
-					queriesToSign = append(queriesToSign, k+"="+v)
-				}
-			}
-		}
-	}
-	// Add query parameters to URI
-	if len(queriesToSign) > 0 {
-		sort.StringSlice(queriesToSign).Sort()
-		uri += "?" + strings.Join(queriesToSign, "&")
-	}
-
-	// Make signature
-	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
-	hash := hmac.New(sha1.New, []byte(SecretKey))
-	hash.Write([]byte(payload))
-	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
-	base64.StdEncoding.Encode(signature, hash.Sum(nil))
-
-	// Set signature in request
-	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
-}

swift/swift.go

@@ -44,12 +44,6 @@ func init() {
 				Help:  "Memset Memstore UK v2",
 				Value: "https://auth.storage.memset.com/v2.0",
 			}},
-		}, {
-			Name: "tenant",
-			Help: "Tenant name - optional",
-		}, {
-			Name: "region",
-			Help: "Region name - optional",
 		},
 		// snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
 		},
@@ -113,15 +107,10 @@ func swiftConnection(name string) (*swift.Connection, error) {
 		return nil, errors.New("auth not found")
 	}
 	c := &swift.Connection{
-		UserName:       userName,
-		ApiKey:         apiKey,
-		AuthUrl:        authUrl,
-		UserAgent:      fs.UserAgent,
-		Tenant:         fs.ConfigFile.MustValue(name, "tenant"),
-		Region:         fs.ConfigFile.MustValue(name, "region"),
-		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
-		Transport:      fs.Config.Transport(),
+		UserName:  userName,
+		ApiKey:    apiKey,
+		AuthUrl:   authUrl,
+		UserAgent: fs.UserAgent,
 	}
 	err := c.Authenticate()
 	if err != nil {
@@ -168,7 +157,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
-func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
+func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
 	fs := &FsObjectSwift{
 		swift:  f,
 		remote: remote,
@@ -190,7 +179,7 @@ func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Obje
 //
 // May return nil if an error occurred
 func (f *FsSwift) NewFsObject(remote string) fs.Object {
-	return f.newFsObjectWithInfo(remote, nil)
+	return f.NewFsObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -212,11 +201,8 @@ func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
 			for i := range objects {
 				object := &objects[i]
 				// FIXME if there are no directories, swift gives back the files for some reason!
-				if directories {
-					if !strings.HasSuffix(object.Name, "/") {
-						continue
-					}
-					object.Name = object.Name[:len(object.Name)-1]
+				if directories && !strings.HasSuffix(object.Name, "/") {
+					continue
 				}
 				if !strings.HasPrefix(object.Name, f.root) {
 					fs.Log(f, "Odd name received %q", object.Name)
@@ -230,7 +216,7 @@ func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
 	})
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(f, "Couldn't read container %q: %s", f.container, err)
+		fs.Log(f, "Couldn't read container %q: %s", f.container, err)
 	}
 }
 
@@ -241,13 +227,13 @@ func (f *FsSwift) List() fs.ObjectsChan {
 		// Return no objects at top level list
 		close(out)
 		fs.Stats.Error()
-		fs.ErrorLog(f, "Can't list objects at root - choose a container using lsd")
+		fs.Log(f, "Can't list objects at root - choose a container using lsd")
 	} else {
 		// List the objects
 		go func() {
 			defer close(out)
 			f.list(false, func(remote string, object *swift.Object) {
-				if fs := f.newFsObjectWithInfo(remote, object); fs != nil {
+				if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
 					out <- fs
 				}
 			})
@@ -266,7 +252,7 @@ func (f *FsSwift) ListDir() fs.DirChan {
 			containers, err := f.c.ContainersAll(nil)
 			if err != nil {
 				fs.Stats.Error()
-				fs.ErrorLog(f, "Couldn't list containers: %v", err)
+				fs.Log(f, "Couldn't list containers: %v", err)
 			} else {
 				for _, container := range containers {
 					out <- &fs.Dir{
@@ -393,14 +379,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
 	err := o.readMetaData()
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to read metadata: %s", err)
+		fs.Log(o, "Failed to read metadata: %s", err)
 		return
 	}
 	o.meta.SetModTime(modTime)
 	err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, o.meta.ObjectHeaders())
 	if err != nil {
 		fs.Stats.Error()
-		fs.ErrorLog(o, "Failed to update remote mtime: %s", err)
+		fs.Log(o, "Failed to update remote mtime: %s", err)
 	}
 }
 
@@ -427,7 +413,6 @@ func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) erro
 		return err
 	}
 	// Read the metadata from the newly created object
-	o.meta = nil // wipe old metadata
 	err = o.readMetaData()
 	return err
 }

swift/swift_test.go

@@ -1,53 +0,0 @@
-// Test Swift filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: go run gen_tests.go or make gen_tests
-package swift_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-	"github.com/ncw/rclone/swift"
-)
-
-func init() {
-	fstests.NilObject = fs.Object((*swift.FsObjectSwift)(nil))
-	fstests.RemoteName = "TestSwift:"
-}
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                  { fstests.TestInit(t) }
-func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
-func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
-func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
-func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
-func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
-func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
-func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
-func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
-func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
-func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
-func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
-func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
-func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
-func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
-func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
-func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
-func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
-func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
-func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
-func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
-func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }