Version 1.01

This was done by making a seekWrapper which wraps an io.Reader with a
basic Seek for code.google.com/p/google-api-go-client/googleapi to
detect the length.  Without this the getReaderSize function reads the
entire file into memory to find its length.

.travis.yml

@@ -0,0 +1,11 @@
+language: go
+
+go:
+  - 1.1.2
+  - 1.2.2
+  - 1.3
+  - tip
+
+script:
+  - go get ./...
+  - go test -v ./...

Makefile

@@ -1,6 +1,6 @@
 TAG := $(shell git describe --tags)
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = "v" . $$_')
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 
 rclone:	*.go */*.go
 	@go version
@@ -50,12 +50,11 @@ tag:
 	perl -lpe 's/VERSION/${NEW_TAG}/g; s/DATE/'`date -I`'/g;' docs/content/downloads.md.in > docs/content/downloads.md
 	git tag $(NEW_TAG)
 	@echo "Add this to changelog in README.md"
-	@echo "  * $(NEW_TAG) - " `date -I`
+	@echo "  * $(NEW_TAG) -" `date -I`
 	@git log $(LAST_TAG)..$(NEW_TAG) --oneline
 	@echo "Then commit the changes"
 	@echo git commit -m "Version $(NEW_TAG)" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
-	echo git tag -f $(LAST_TAG)
-
+	git tag -f $(LAST_TAG)

README.md

@@ -1,6 +1,6 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Apr 24, 2014
+% Jul 7, 2014
 
 Rclone
 ======
@@ -214,7 +214,13 @@ Bugs
 
 Changelog
 ---------
-
+  * v1.01 - 2014-07-04
+    * drive: fix transfer of big files using up lots of memory
+  * v1.00 - 2014-07-03
+    * drive: fix whole second dates
+  * v0.99 - 2014-06-26
+    * Fix --dry-run not working
+    * Make compatible with go 1.1
   * v0.98 - 2014-05-30
     * s3: Treat missing Content-Length as 0 for some ceph installations
     * rclonetest: add file with a space in

docs/content/downloads.md

@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2014-05-30"
+date: "2014-07-04"
 ---
 
-Rclone Download v0.98
+Rclone Download v1.01
 =====================
 
   * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-windows-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-windows-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-windows-amd64.zip)
   * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-osx-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-osx-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-osx-amd64.zip)
   * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.98-linux-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-linux-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-linux-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.01-linux-arm.zip)
   * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.98-freebsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-freebsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-freebsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.01-freebsd-arm.zip)
   * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.98-netbsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-netbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-netbsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.01-netbsd-arm.zip)
   * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.98-openbsd-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-openbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.01-openbsd-amd64.zip)
   * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.98-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.01-plan9-386.zip)
 
 Older downloads can be found [here](http://downloads.rclone.org/)

drive/drive.go

@@ -22,6 +22,7 @@ import (
 	"log"
 	"mime"
 	"net/http"
+	"os"
 	"path"
 	"strings"
 	"sync"
@@ -38,6 +39,8 @@ const (
 	rcloneClientId     = "202264815644.apps.googleusercontent.com"
 	rcloneClientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
 	driveFolderType    = "application/vnd.google-apps.folder"
+	RFC3339In          = time.RFC3339
+	RFC3339Out         = "2006-01-02T15:04:05.000000000Z07:00"
 )
 
 // Globals
@@ -674,7 +677,7 @@ func (f *FsDrive) ListDir() fs.DirChan {
 					Bytes: -1,
 					Count: -1,
 				}
-				dir.When, _ = time.Parse(time.RFC3339, item.ModifiedDate)
+				dir.When, _ = time.Parse(RFC3339In, item.ModifiedDate)
 				out <- dir
 				return false
 			})
@@ -687,6 +690,30 @@ func (f *FsDrive) ListDir() fs.DirChan {
 	return out
 }
 
+// seekWrapper wraps an io.Reader with a basic Seek for
+// code.google.com/p/google-api-go-client/googleapi
+// to detect the length (see getReaderSize function)
+type seekWrapper struct {
+	in   io.Reader
+	size int64
+}
+
+// Read bytes from the object - see io.Reader
+func (file *seekWrapper) Read(p []byte) (n int, err error) {
+	return file.in.Read(p)
+}
+
+// Seek - minimal implementation for Google Drive's length detection
+func (file *seekWrapper) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case os.SEEK_CUR:
+		return 0, nil
+	case os.SEEK_END:
+		return file.size, nil
+	}
+	return 0, nil
+}
+
 // Put the object
 //
 // This assumes that the object doesn't not already exists - if you
@@ -711,7 +738,7 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
 	if mimeType == "" {
 		mimeType = "application/octet-stream"
 	}
-	modifiedDate := modTime.Format(time.RFC3339Nano)
+	modifiedDate := modTime.Format(RFC3339Out)
 
 	// Define the metadata for the file we are going to create.
 	info := &drive.File{
@@ -723,6 +750,7 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
 	}
 
 	// Make the API request to upload metadata and file data.
+	in = &seekWrapper{in: in, size: size}
 	info, err = f.svc.Files.Insert(info).Media(in).Do()
 	if err != nil {
 		return o, fmt.Errorf("Upload failed: %s", err)
@@ -864,7 +892,7 @@ func (o *FsObjectDrive) ModTime() time.Time {
 		fs.Log(o, "Failed to read metadata: %s", err)
 		return time.Now()
 	}
-	modTime, err := time.Parse(time.RFC3339, o.modifiedDate)
+	modTime, err := time.Parse(RFC3339In, o.modifiedDate)
 	if err != nil {
 		fs.Log(o, "Failed to read mtime from object: %s", err)
 		return time.Now()
@@ -882,7 +910,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
 	}
 	// New metadata
 	info := &drive.File{
-		ModifiedDate: modTime.Format(time.RFC3339Nano),
+		ModifiedDate: modTime.Format(RFC3339Out),
 	}
 	// Set modified date
 	_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
@@ -920,10 +948,11 @@ func (o *FsObjectDrive) Open() (in io.ReadCloser, err error) {
 func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) error {
 	info := &drive.File{
 		Id:           o.id,
-		ModifiedDate: modTime.Format(time.RFC3339Nano),
+		ModifiedDate: modTime.Format(RFC3339Out),
 	}
 
 	// Make the API request to upload metadata and file data.
+	in = &seekWrapper{in: in, size: size}
 	info, err := o.drive.svc.Files.Update(info.Id, info).SetModifiedDate(true).Media(in).Do()
 	if err != nil {
 		return fmt.Errorf("Update failed: %s", err)

fs/config.go

@@ -70,10 +70,10 @@ func LoadConfig() {
 	// FIXME read these from the config file too
 	Config.Verbose = *verbose
 	Config.Quiet = *quiet
-	Config.Quiet = *dryRun
 	Config.ModifyWindow = *modifyWindow
 	Config.Checkers = *checkers
 	Config.Transfers = *transfers
+	Config.DryRun = *dryRun
 
 	ConfigPath = *configFile
 
@@ -147,7 +147,7 @@ func Command(commands []string) byte {
 		if len(result) != 1 {
 			continue
 		}
-		i := strings.IndexByte(optString, result[0])
+		i := strings.Index(optString, string(result[0]))
 		if i >= 0 {
 			return result[0]
 		}

fs/operations.go

@@ -178,7 +178,11 @@ func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
 	for pair := range in {
 		src := pair.src
 		Stats.Transferring(src)
-		Copy(fdst, pair.dst, src)
+		if Config.DryRun {
+			Debug(src, "Not copying as --dry-run")
+		} else {
+			Copy(fdst, pair.dst, src)
+		}
 		Stats.DoneTransferring(src)
 	}
 }

notes.txt

@@ -54,11 +54,11 @@ Making a release
   * ./rclonetest /tmp/z
   * cd ..
   * make tag
-  * edit README.md Changelog
+  * edit README.md
   * git commit version.go rclonetest/version.go README.md docs/content/downloads.md
   * make retag
   * . ~/bin/go-cross
   * make cross
   * make upload
   * make upload_website
-  * git push --tags
+  * git push --tags origin master

rclonetest/rclonetest.go

@@ -145,13 +145,18 @@ func TestMkdir(flocal, fremote fs.Fs) {
 	CheckListing(fremote, items)
 }
 
-var t1 = Time("2001-02-03T04:05:06.999999999Z")
+var t1 = Time("2001-02-03T04:05:06.499999999Z")
 var t2 = Time("2011-12-25T12:59:59.123456789Z")
+var t3 = Time("2011-12-30T12:59:59.000000000Z")
 
 func TestCopy(flocal, fremote fs.Fs) {
 	WriteFile("empty space", "", t1)
 
+	// Check dry run is working
+	log.Printf("Copy with --dry-run")
+	fs.Config.DryRun = true
 	err := fs.Sync(fremote, flocal, false)
+	fs.Config.DryRun = false
 	if err != nil {
 		log.Fatalf("Copy failed: %v", err)
 	}
@@ -159,6 +164,18 @@ func TestCopy(flocal, fremote fs.Fs) {
 	items := []Item{
 		{Path: "empty space", Size: 0, ModTime: t1, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
 	}
+
+	CheckListing(flocal, items)
+	CheckListing(fremote, []Item{})
+
+	// Now without dry run
+
+	log.Printf("Copy")
+	err = fs.Sync(fremote, flocal, false)
+	if err != nil {
+		log.Fatalf("Copy failed: %v", err)
+	}
+
 	CheckListing(flocal, items)
 	CheckListing(fremote, items)
 }
@@ -182,14 +199,14 @@ func TestSync(flocal, fremote fs.Fs) {
 	// ------------------------------------------------------------
 
 	log.Printf("Sync after adding a file")
-	WriteFile("potato", "------------------------------------------------------------", t1)
+	WriteFile("potato", "------------------------------------------------------------", t3)
 	err = fs.Sync(fremote, flocal, true)
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
 	items = []Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
 	}
 	CheckListing(flocal, items)
 	CheckListing(fremote, items)
@@ -197,36 +214,51 @@ func TestSync(flocal, fremote fs.Fs) {
 	// ------------------------------------------------------------
 
 	log.Printf("Sync after changing a file's size only")
-	WriteFile("potato", "smaller but same date", t1)
+	WriteFile("potato", "smaller but same date", t3)
 	err = fs.Sync(fremote, flocal, true)
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
 	items = []Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t1, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
 	}
 	CheckListing(flocal, items)
 	CheckListing(fremote, items)
 
 	// ------------------------------------------------------------
 
-	log.Printf("Sync after removing a file")
+	log.Printf("Sync after removing a file and adding a file --dry-run")
+	WriteFile("potato2", "------------------------------------------------------------", t1)
 	err = os.Remove(localName + "/potato")
 	if err != nil {
 		log.Fatalf("Remove failed: %v", err)
 	}
+	fs.Config.DryRun = true
 	err = fs.Sync(fremote, flocal, true)
+	fs.Config.DryRun = false
 	if err != nil {
 		log.Fatalf("Sync failed: %v", err)
 	}
 
+	before := []Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+	}
 	items = []Item{
 		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	CheckListing(flocal, items)
+	CheckListing(fremote, before)
+
+	log.Printf("Sync after removing a file and adding a file")
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		log.Fatalf("Sync failed: %v", err)
 	}
 	CheckListing(flocal, items)
 	CheckListing(fremote, items)
-
 }
 
 func TestLs(flocal, fremote fs.Fs) {

rclonetest/version.go

@@ -1,3 +1,3 @@
 package main
 
-const Version = "v0.98"
+const Version = "v1.01"

version.go

@@ -1,3 +1,3 @@
 package main
 
-const Version = "v0.98"
+const Version = "v1.01"