Version v1.16

The symtom was one of these two on upload of files only
  * panic: d.nx != 0 in crypto/md5.(*digest).checkSum
  * read tcp: i/o timeout

It turned out to be a combination of two upstream bugs

  * 5a2187309e
  * https://groups.google.com/forum/#!topic/golang-dev/0Nl6k5Sj6UU

This commit contains a work-around for the second problem, I've fixed
the first and had the change accepted upstream.

docs/content/changelog.md

@@ -1,12 +1,15 @@
 ---
 title: "Documentation"
 description: "Rclone Changelog"
-date: "2015-06-06"
+date: "2015-06-09"
 ---
 
 Changelog
 ---------
 
+  * v1.16 - 2015-06-09
+    * Fix uploading big files which was causing timeouts or panics
+    * Don't check md5sum after download with --size-only
   * v1.15 - 2015-06-06
     * Add --checksum flag to only discard transfers by MD5SUM - thanks Alex Couper
     * Implement --size-only flag to sync on size not checksum & modtime

docs/content/downloads.md

@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2015-06-06"
+date: "2015-06-09"
 ---
 
-Rclone Download v1.15
+Rclone Download v1.16
 =====================
 
   * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-windows-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-windows-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-windows-amd64.zip)
   * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-osx-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-osx-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-osx-amd64.zip)
   * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.15-linux-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-linux-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-linux-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.16-linux-arm.zip)
   * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.15-freebsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-freebsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-freebsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.16-freebsd-arm.zip)
   * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.15-netbsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-netbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-netbsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.16-netbsd-arm.zip)
   * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.15-openbsd-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-openbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.16-openbsd-amd64.zip)
   * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.15-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.16-plan9-386.zip)
 
 Older downloads can be found [here](http://downloads.rclone.org/)

fs/accounting.go

@@ -187,6 +187,12 @@ func (s *StatsInfo) DoneTransferring(o Object) {
 
 // Account limits and accounts for one transfer
 type Account struct {
+	// The mutex is to make sure Read() and Close() aren't called
+	// concurrently.  Unfortunately the persistent connection loop
+	// in http transport calls Read() after Do() returns on
+	// CancelRequest so this race can happen when it apparently
+	// shouldn't.
+	mu    sync.Mutex
 	in    io.ReadCloser
 	bytes int64
 }
@@ -200,6 +206,8 @@ func NewAccount(in io.ReadCloser) *Account {
 
 // Read bytes from the object - see io.Reader
 func (file *Account) Read(p []byte) (n int, err error) {
+	file.mu.Lock()
+	defer file.mu.Unlock()
 	n, err = file.in.Read(p)
 	file.bytes += int64(n)
 	Stats.Bytes(int64(n))
@@ -215,6 +223,8 @@ func (file *Account) Read(p []byte) (n int, err error) {
 
 // Close the object
 func (file *Account) Close() error {
+	file.mu.Lock()
+	defer file.mu.Unlock()
 	// FIXME do something?
 	return file.in.Close()
 }

fs/operations.go

@@ -193,21 +193,23 @@ tryAgain:
 	}
 
 	// Verify md5sums are the same after transfer - ignoring blank md5sums
-	srcMd5sum, md5sumErr := src.Md5sum()
-	if md5sumErr != nil {
-		Stats.Error()
-		Log(src, "Failed to read md5sum: %s", md5sumErr)
-	} else if srcMd5sum != "" {
-		dstMd5sum, md5sumErr := dst.Md5sum()
+	if !Config.SizeOnly {
+		srcMd5sum, md5sumErr := src.Md5sum()
 		if md5sumErr != nil {
 			Stats.Error()
-			Log(dst, "Failed to read md5sum: %s", md5sumErr)
-		} else if dstMd5sum != "" && srcMd5sum != dstMd5sum {
-			Stats.Error()
-			err = fmt.Errorf("Corrupted on transfer: md5sums differ %q vs %q", srcMd5sum, dstMd5sum)
-			Log(dst, "%s", err)
-			removeFailedCopy(dst)
-			return
+			Log(src, "Failed to read md5sum: %s", md5sumErr)
+		} else if srcMd5sum != "" {
+			dstMd5sum, md5sumErr := dst.Md5sum()
+			if md5sumErr != nil {
+				Stats.Error()
+				Log(dst, "Failed to read md5sum: %s", md5sumErr)
+			} else if dstMd5sum != "" && srcMd5sum != dstMd5sum {
+				Stats.Error()
+				err = fmt.Errorf("Corrupted on transfer: md5sums differ %q vs %q", srcMd5sum, dstMd5sum)
+				Log(dst, "%s", err)
+				removeFailedCopy(dst)
+				return
+			}
 		}
 	}
 

fs/version.go

@@ -1,3 +1,3 @@
 package fs
 
-const Version = "v1.15"
+const Version = "v1.16"