Version v1.14

The breaking change is: google-api-go-generator: remove underscores from identifiers.

README.md

@@ -262,6 +262,10 @@ Bugs
 
 Changelog
 ---------
+  * v1.14 - 2015-05-21
+    * local: fix encoding of non utf-8 file names - fixes a duplicate file problem
+    * drive: docs about rate limiting
+    * google cloud storage: Fix compile after API change in "google.golang.org/api/storage/v1"
   * v1.13 - 2015-05-10
     * Revise documentation (especially sync)
     * Implement --timeout and --conntimeout

docs/content/downloads.md

@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2015-05-10"
+date: "2015-05-21"
 ---
 
-Rclone Download v1.13
+Rclone Download v1.14
 =====================
 
   * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-windows-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-windows-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-windows-amd64.zip)
   * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-osx-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-osx-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-osx-amd64.zip)
   * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-linux-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-linux-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.13-linux-arm.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.14-linux-arm.zip)
   * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-freebsd-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-freebsd-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.13-freebsd-arm.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.14-freebsd-arm.zip)
   * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-netbsd-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-netbsd-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.13-netbsd-arm.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.14-netbsd-arm.zip)
   * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-openbsd-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.13-openbsd-amd64.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.14-openbsd-amd64.zip)
   * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.13-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.14-plan9-386.zip)
 
 Older downloads can be found [here](http://downloads.rclone.org/)

docs/content/drive.md

@@ -86,3 +86,11 @@ was
 
   * They are deleted after 30 days or 100 revisions (whatever comes first).
   * They do not count towards a user storage quota.
+
+Limitations
+-----------
+
+Drive has quite a lot of rate limiting.  This causes rclone to be
+limited to transferring about 2 files per second only.  Individual
+files may be transferred much faster at 100s of MBytes/s but lots of
+small files can take a long time.

docs/content/local.md

@@ -23,3 +23,18 @@ Rclone reads and writes the modified time using an accuracy determined by
 the OS.  Typically this is 1ns on Linux, 10 ns on Windows and 1 Second
 on OS X.
 
+Filenames
+---------
+
+Filenames are expected to be encoded in UTF-8 on disk.  This is the
+normal case for Windows and OS X.  There is a bit more uncertainty in
+the Linux world, but new distributions will have UTF-8 encoded files
+names.
+
+If an invalid (non-UTF8) filename is read, the invalid caracters will
+be replaced with the unicode replacement character, '<27>'.  `rclone`
+will emit a warning in this case, eg
+
+```
+Local file system at .: Replacing invalid UTF-8 characters in "gro\xdf"
+```

fs/version.go

@@ -1,3 +1,3 @@
 package fs
 
-const Version = "v1.13"
+const Version = "v1.14"

googlecloudstorage/googlecloudstorage.go

@@ -42,7 +42,7 @@ const (
 var (
 	// Description of how to auth for this app
 	storageAuth = &googleauth.Auth{
-		Scope:               storage.DevstorageFull_controlScope,
+		Scope:               storage.DevstorageFullControlScope,
 		DefaultClientId:     rcloneClientId,
 		DefaultClientSecret: rcloneClientSecret,
 	}

local/local.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"github.com/ncw/rclone/fs"
 )
@@ -32,14 +33,15 @@ func init() {
 
 // FsLocal represents a local filesystem rooted at root
 type FsLocal struct {
-	root        string        // The root directory
+	root        string              // The root directory
-	precisionOk sync.Once     // Whether we need to read the precision
+	precisionOk sync.Once           // Whether we need to read the precision
-	precision   time.Duration // precision of local filesystem
+	precision   time.Duration       // precision of local filesystem
+	warned      map[string]struct{} // whether we have warned about this string
 }
 
 // FsObjectLocal represents a local filesystem object
 type FsObjectLocal struct {
-	local  fs.Fs       // The Fs this object is part of
+	local  *FsLocal    // The Fs this object is part of
 	remote string      // The remote path
 	path   string      // The local path
 	info   os.FileInfo // Interface for file info (always present)
@@ -51,7 +53,10 @@ type FsObjectLocal struct {
 // NewFs contstructs an FsLocal from the path
 func NewFs(name, root string) (fs.Fs, error) {
 	root = filepath.ToSlash(path.Clean(root))
-	f := &FsLocal{root: root}
+	f := &FsLocal{
+		root:   root,
+		warned: make(map[string]struct{}),
+	}
 	// Check to see if this points to a file
 	fi, err := os.Lstat(f.root)
 	if err == nil && fi.Mode().IsRegular() {
@@ -134,6 +139,20 @@ func (f *FsLocal) List() fs.ObjectsChan {
 	return out
 }
 
+// CleanUtf8 makes string a valid UTF-8 string
+//
+// Any invalid UTF-8 characters will be replaced with utf8.RuneError
+func (f *FsLocal) cleanUtf8(name string) string {
+	if utf8.ValidString(name) {
+		return name
+	}
+	if _, ok := f.warned[name]; !ok {
+		fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
+		f.warned[name] = struct{}{}
+	}
+	return string([]rune(name))
+}
+
 // Walk the path returning a channel of FsObjects
 func (f *FsLocal) ListDir() fs.DirChan {
 	out := make(fs.DirChan, fs.Config.Checkers)
@@ -147,7 +166,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
 			for _, item := range items {
 				if item.IsDir() {
 					dir := &fs.Dir{
-						Name:  item.Name(),
+						Name:  f.cleanUtf8(item.Name()),
 						When:  item.ModTime(),
 						Bytes: 0,
 						Count: 0,
@@ -294,7 +313,7 @@ func (o *FsObjectLocal) String() string {
 
 // Return the remote path
 func (o *FsObjectLocal) Remote() string {
-	return o.remote
+	return o.local.cleanUtf8(o.remote)
 }
 
 // Md5sum calculates the Md5sum of a file returning a lowercase hex string