Version v1.10

s3: list an unlimited number of items - fixes #22
Fix config loop - fixes #25
2026-01-12 21:43:13 +00:00 · 2015-02-12 18:00:20 +00:00 · 2015-02-10 17:58:29 +00:00 · 2015-02-10 16:48:04 +00:00
5 changed files with 64 additions and 47 deletions
--- a/README.md
+++ b/README.md
@@ -259,6 +259,9 @@ Bugs

 Changelog
 ---------
+  * v1.10 - 2015-02-12
+    * s3: list an unlimited number of items
+    * Fix getting stuck in the configurator
  * v1.09 - 2015-02-07
    * windows: Stop drive letters (eg C:) getting mixed up with remotes (eg drive:)
    * local: Fix directory separators on Windows
--- a/docs/content/downloads.md
+++ b/docs/content/downloads.md
@@ -2,34 +2,34 @@
 title: "Rclone downloads"
 description: "Download rclone binaries for your OS."
 type: page
-date: "2015-02-07"
+date: "2015-02-12"
 ---

-Rclone Download v1.09
+Rclone Download v1.10
 =====================

  * Windows
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-windows-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-windows-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-windows-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-windows-amd64.zip)
  * OSX
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-osx-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-osx-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-osx-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-osx-amd64.zip)
  * Linux
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-linux-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-linux-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.09-linux-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-linux-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-linux-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.10-linux-arm.zip)
  * FreeBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-freebsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-freebsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.09-freebsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-freebsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-freebsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.10-freebsd-arm.zip)
  * NetBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-netbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-netbsd-amd64.zip)
-    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.09-netbsd-arm.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-netbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-netbsd-amd64.zip)
+    * [ARM - 32 Bit](http://downloads.rclone.org/rclone-v1.10-netbsd-arm.zip)
  * OpenBSD
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-openbsd-386.zip)
-    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.09-openbsd-amd64.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-openbsd-386.zip)
+    * [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v1.10-openbsd-amd64.zip)
  * Plan 9
-    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.09-plan9-386.zip)
+    * [386 - 32 Bit](http://downloads.rclone.org/rclone-v1.10-plan9-386.zip)

 Older downloads can be found [here](http://downloads.rclone.org/)
--- a/fs/config.go
+++ b/fs/config.go
@@ -330,6 +330,7 @@ func EditConfig() {
 			name := ChooseRemote()
 			EditRemote(name)
 		case 'n':
+		nameLoop:
 			for {
 				fmt.Printf("name> ")
 				name := ReadLine()
@@ -340,7 +341,7 @@ func EditConfig() {
 					fmt.Printf("Can't use %q as it can be confused a drive letter\n", name)
 				default:
 					NewRemote(name)
-					break
+					break nameLoop
 				}
 			}
 		case 'd':
--- a/fs/version.go
+++ b/fs/version.go
@@ -1,3 +1,3 @@
 package fs

-const Version = "v1.09"
+const Version = "v1.10"
--- a/s3/s3.go
+++ b/s3/s3.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"mime"
 	"net/http"
 	"path"
@@ -101,7 +102,8 @@ func init() {

 // Constants
 const (
-	metaMtime = "X-Amz-Meta-Mtime" // the meta key to store mtime in
+	metaMtime     = "X-Amz-Meta-Mtime" // the meta key to store mtime in
+	listChunkSize = 1024               // number of items to read at once
 )

 // FsS3 represents a remote s3 server
@@ -267,36 +269,47 @@ func (f *FsS3) list(directories bool, fn func(string, *s3.Key)) {
 	if directories {
 		delimiter = "/"
 	}
-	// FIXME need to implement ALL loop
-	objects, err := f.b.List(f.root, delimiter, "", 10000)
-	if err != nil {
-		fs.Stats.Error()
-		fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
-	} else {
-		rootLength := len(f.root)
-		if directories {
-			for _, remote := range objects.CommonPrefixes {
-				if !strings.HasPrefix(remote, f.root) {
-					fs.Log(f, "Odd name received %q", remote)
-					continue
-				}
-				remote := remote[rootLength:]
-				if strings.HasSuffix(remote, "/") {
-					remote = remote[:len(remote)-1]
-				}
-				fn(remote, &s3.Key{Key: remote})
-			}
+	marker := ""
+	for {
+		objects, err := f.b.List(f.root, delimiter, marker, listChunkSize)
+		if err != nil {
+			fs.Stats.Error()
+			fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
 		} else {
-			for i := range objects.Contents {
-				object := &objects.Contents[i]
-				if !strings.HasPrefix(object.Key, f.root) {
-					fs.Log(f, "Odd name received %q", object.Key)
-					continue
+			rootLength := len(f.root)
+			if directories {
+				for _, remote := range objects.CommonPrefixes {
+					if !strings.HasPrefix(remote, f.root) {
+						fs.Log(f, "Odd name received %q", remote)
+						continue
+					}
+					remote := remote[rootLength:]
+					if strings.HasSuffix(remote, "/") {
+						remote = remote[:len(remote)-1]
+					}
+					fn(remote, &s3.Key{Key: remote})
+				}
+			} else {
+				for i := range objects.Contents {
+					object := &objects.Contents[i]
+					if !strings.HasPrefix(object.Key, f.root) {
+						fs.Log(f, "Odd name received %q", object.Key)
+						continue
+					}
+					remote := object.Key[rootLength:]
+					fn(remote, object)
 				}
-				remote := object.Key[rootLength:]
-				fn(remote, object)
 			}
 		}
+		if !objects.IsTruncated {
+			break
+		}
+		// Use NextMarker if set, otherwise use last Key
+		marker = objects.NextMarker
+		if marker == "" {
+			marker = objects.Contents[len(objects.Contents)-1].Key
+		}
+		log.Printf("retry with marker = %q", marker)
 	}
 }
Author	SHA1	Message	Date
Nick Craig-Wood	cf208ad21b	Version v1.10	2015-02-12 18:00:20 +00:00
Nick Craig-Wood	0faed16899	s3: list an unlimited number of items - fixes #22	2015-02-10 17:58:29 +00:00
Nick Craig-Wood	8d1c0ad07c	Fix config loop - fixes #25	2015-02-10 16:48:04 +00:00