local: put debug statements in on Read FIXME WIP

.circleci/config.yml

@@ -1,4 +1,3 @@
----
 version: 2
 
 jobs:
@@ -42,9 +41,7 @@ jobs:
     - run:
         name: Upload artifacts
         command: |
-          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
-              make circleci_upload
-          fi
+            make circleci_upload
 
     - store_artifacts:
         path: /tmp/rclone.dist

.golangci.yml

@@ -1,30 +0,0 @@
-# golangci-lint configuration options
-
-run:
-  build-tags:
-    - cmount
-
-linters:
-  enable:
-    - deadcode
-    - errcheck
-    - goimports
-    - golint
-    - ineffassign
-    - structcheck
-    - varcheck
-    - govet
-    - unconvert
-    #- prealloc
-    #- maligned
-  disable-all: true
-
-issues:
-  # Enable some lints excluded by default
-  exclude-use-default: false
-
-  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
-
-  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
-  max-same-issues: 0

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.travis.yml

@@ -1,103 +1,52 @@
----
 language: go
 sudo: required
 dist: trusty
 os:
-  - linux
+- linux
+go:
+- 1.8.x
+- 1.9.x
+- 1.10.x
+- 1.11.x
+- tip
 go_import_path: github.com/ncw/rclone
 before_install:
-  - git fetch --unshallow --tags
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-      choco install -y winfsp zip make
-      cd ../..                  # fix crlf in git checkout
-      mv $TRAVIS_REPO_SLUG _old
-      git config --global core.autocrlf false
-      git clone _old $TRAVIS_REPO_SLUG
-      cd $TRAVIS_REPO_SLUG
-    fi
+- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
+- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
 install:
-  - make vars
+- git fetch --unshallow --tags
+- make vars
+- make build_dep
+script:
+- make check
+- make quicktest
+- make compile_all
 env:
   global:
     - GOTAGS=cmount
-    - GO111MODULE=off
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
     - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-      - fuse
-      - libfuse-dev
-      - rpm
-      - pkg-config
+    - fuse
+    - libfuse-dev
+    - rpm
+    - pkg-config
 cache:
   directories:
     - $HOME/.cache/go-build
 matrix:
   allow_failures:
-    - go: tip
+  - go: tip
   include:
-    - go: 1.8.x
-      script:
-        - make quicktest
-    - go: 1.9.x
-      script:
-        - make quicktest
-    - go: 1.10.x
-      script:
-        - make quicktest
-    - go: 1.11.x
-      script:
-        - make quicktest
-    - go: 1.12.x
-      env:
-        - GOTAGS=cmount
-      script:
-        - make build_dep
-        - make check
-        - make quicktest
-        - make racequicktest
-        - make compile_all
-    - os: osx
-      go: 1.12.x
-      env:
-        - GOTAGS=  # cmount doesn't work on osx travis for some reason
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-      script:
-        - make
-        - make quicktest
-        - make racequicktest
-    # - os: windows
-    #   go: 1.12.x
-    #   env:
-    #     - GOTAGS=cmount
-    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
-    #   #filter_secrets: false     # works around a problem with secrets under windows
-    #   cache:
-    #     directories:
-    #       - ${LocalAppData}/go-build
-    #   script:
-    #     - make
-    #     - make quicktest
-    #     - make racequicktest
-    - go: tip
-      script:
-        - make quicktest
-
+  - os: osx
+    go: 1.11.x
+    env: GOTAGS=""
+    cache:
+      directories:
+        - $HOME/Library/Caches/go-build
 deploy:
   provider: script
   script: make travis_beta
@@ -105,5 +54,5 @@ deploy:
   on:
     repo: ncw/rclone
     all_branches: true
-    go: 1.12.x
-    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"
+    go: 1.11.x
+    condition: $TRAVIS_PULL_REQUEST == false

Makefile

@@ -11,12 +11,14 @@ ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	BRANCH_PATH :=
 endif
 TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 ifneq ($(TAG),$(LAST_TAG))
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+# Run full tests if go >= go1.11
+FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -40,6 +42,7 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
+	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -54,22 +57,38 @@ test:	rclone
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-
-racequicktest:
+ifdef FULL_TESTS
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
+endif
 
 # Do source code quality checks
 check:	rclone
-	@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
-	@# see: https://github.com/golangci/golangci-lint/issues/204
-	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
-	@golangci-lint run ./...
-	@echo "-- END CODE QUALITY REPORT ---------------------------------"
+ifdef FULL_TESTS
+	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	errcheck $(BUILDTAGS) ./...
+	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
+	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+else
+	@echo Skipping source quality tests as version of go too old
+endif
+
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+ifdef FULL_TESTS
+	go get -u github.com/kisielk/errcheck
+	go get -u golang.org/x/tools/cmd/goimports
+	go get -u golang.org/x/lint/golint
+endif
 
 # Get the release dependencies
 release_dep:
@@ -153,7 +172,11 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
+ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+else
+	@echo Skipping compile all as version of go too old
+endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -173,15 +196,10 @@ BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
-ifeq ($(TRAVIS_OS_NAME),windows)
-# BUILD_FLAGS := -include "^windows/" -cgo
-# 386 doesn't build yet
-	BUILD_FLAGS := -include "^windows/amd64" -cgo
-endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)

README.md

@@ -36,7 +36,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
-  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
@@ -45,14 +44,13 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
+  * Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
-  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -63,13 +61,13 @@ Please see [the full list of all storage providers and their features](https://r
 
 ## Features
 
-  * MD5/SHA-1 hashes checked at all times for file integrity
+  * MD5/SHA1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, e.g. two different cloud accounts
+  * Can sync to and from network, eg two different cloud accounts
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))

backend/alias/alias.go

@@ -30,7 +30,7 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs constructs an Fs from the path.
+// NewFs contstructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {

backend/alias/alias_internal_test.go

@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}

backend/all/all.go

@@ -16,7 +16,6 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/jottacloud"
-	_ "github.com/ncw/rclone/backend/koofr"
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"

backend/amazonclouddrive/amazonclouddrive.go

@@ -155,7 +155,7 @@ type Fs struct {
 	noAuthClient *http.Client       // unauthenticated http client
 	root         string             // the path we are working on
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	trueRootID   string             // ID of true root directory
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
@@ -273,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:         root,
 		opt:          *opt,
 		c:            c,
-		pacer:        fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
+		pacer:        pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
 		noAuthClient: fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{

backend/azureblob/azureblob.go

@@ -77,7 +77,7 @@ func init() {
 		}, {
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
-			Default:  defaultUploadCutoff,
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -85,7 +85,7 @@ func init() {
 
 Note that this is stored in memory and there may be up to
 "--transfers" chunks stored at once in memory.`,
-			Default:  defaultChunkSize,
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}, {
 			Name: "list_chunk",
@@ -144,7 +144,7 @@ type Fs struct {
 	containerOKMu    sync.Mutex            // mutex to protect container OK
 	containerOK      bool                  // true if we have created the container
 	containerDeleted bool                  // true if we have deleted the container
-	pacer            *fs.Pacer             // To pace and retry the API calls
+	pacer            *pacer.Pacer          // To pace and retry the API calls
 	uploadToken      *pacer.TokenDispenser // control concurrency
 }
 
@@ -307,7 +307,7 @@ func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline
 	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -347,7 +347,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:         *opt,
 		container:   container,
 		root:        directory,
-		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetPacer(pacer.S3Pacer),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 		client:      fshttp.NewClient(fs.Config),
 	}
@@ -392,7 +392,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
 			}
 
-			f.container = parts.ContainerName
+			container = parts.ContainerName
 			containerURL = azblob.NewContainerURL(*u, pipeline)
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -754,35 +754,6 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
-// Check if the container exists
-//
-// NB this can return incorrect results if called immediately after container deletion
-func (f *Fs) dirExists() (bool, error) {
-	options := azblob.ListBlobsSegmentOptions{
-		Details: azblob.BlobListingDetails{
-			Copy:             false,
-			Metadata:         false,
-			Snapshots:        false,
-			UncommittedBlobs: false,
-			Deleted:          false,
-		},
-		MaxResults: 1,
-	}
-	err := f.pacer.Call(func() (bool, error) {
-		ctx := context.Background()
-		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
-		return f.shouldRetry(err)
-	})
-	if err == nil {
-		return true, nil
-	}
-	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
-	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
-		return false, nil
-	}
-	return false, err
-}
-
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	f.containerOKMu.Lock()
@@ -790,15 +761,6 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
-	if !f.containerDeleted {
-		exists, err := f.dirExists()
-		if err == nil {
-			f.containerOK = exists
-		}
-		if err != nil || exists {
-			return err
-		}
-	}
 
 	// now try to create the container
 	err := f.pacer.Call(func() (bool, error) {
@@ -1038,7 +1000,7 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetProper
 	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
 	o.mimeType = info.ContentType()
 	o.size = size
-	o.modTime = info.LastModified()
+	o.modTime = time.Time(info.LastModified())
 	o.accessTier = azblob.AccessTierType(info.AccessTier())
 	o.setMetadata(metadata)
 
@@ -1104,6 +1066,12 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
+// timeString returns modTime as the number of milliseconds
+// elapsed since January 1, 1970 UTC as a decimal string.
+func timeString(modTime time.Time) string {
+	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
+}
+
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1386,16 +1354,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	blob := o.getBlobReference()
 	httpHeaders := azblob.BlobHTTPHeaders{}
 	httpHeaders.ContentType = fs.MimeType(o)
-	// Compute the Content-MD5 of the file, for multiparts uploads it
-	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
-	// 		 in order to validate its integrity during transport
-	if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
-		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-		if err == nil {
-			httpHeaders.ContentMD5 = sourceMD5bytes
-		} else {
-			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+	// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
+	// MD5 only for PutBlob requests
+	if size < int64(o.fs.opt.UploadCutoff) {
+		if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				httpHeaders.ContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
 		}
 	}
 

backend/b2/api/types.go

@@ -17,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal satisfies the Fatal interface
+// Fatal statisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
 	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is uninitialized
+// IsZero returns true if the timestamp is unitialised
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }

backend/b2/b2.go

@@ -108,7 +108,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 Files above this size will be uploaded in chunks of "--b2-chunk-size".
 
 This value should be set no larger than 4.657GiB (== 5GB).`,
-			Default:  defaultUploadCutoff,
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -117,21 +117,8 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
 When uploading large files, chunk the file into this size.  Note that
 these chunks are buffered in memory and there might a maximum of
 "--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimum size.`,
-			Default:  defaultChunkSize,
-			Advanced: true,
-		}, {
-			Name:     "disable_checksum",
-			Help:     `Disable checksums for large (> upload cutoff) files`,
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name: "download_url",
-			Help: `Custom endpoint for downloads.
-
-This is usually set to a Cloudflare CDN URL as Backblaze offers
-free egress for data downloaded through the Cloudflare network.
-Leave blank if you want to use the endpoint provided by Backblaze.`,
+minimim size.`,
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}},
 	})
@@ -139,16 +126,14 @@ Leave blank if you want to use the endpoint provided by Backblaze.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account         string        `config:"account"`
-	Key             string        `config:"key"`
-	Endpoint        string        `config:"endpoint"`
-	TestMode        string        `config:"test_mode"`
-	Versions        bool          `config:"versions"`
-	HardDelete      bool          `config:"hard_delete"`
-	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
-	DisableCheckSum bool          `config:"disable_checksum"`
-	DownloadURL     string        `config:"download_url"`
+	Account      string        `config:"account"`
+	Key          string        `config:"key"`
+	Endpoint     string        `config:"endpoint"`
+	TestMode     string        `config:"test_mode"`
+	Versions     bool          `config:"versions"`
+	HardDelete   bool          `config:"hard_delete"`
+	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
 }
 
 // Fs represents a remote b2 server
@@ -167,7 +152,7 @@ type Fs struct {
 	uploadMu      sync.Mutex                   // lock for upload variable
 	uploads       []*api.GetUploadURLResponse  // result of get upload URL calls
 	authMu        sync.Mutex                   // lock for authorizing the account
-	pacer         *fs.Pacer                    // To pace and retry the API calls
+	pacer         *pacer.Pacer                 // To pace and retry the API calls
 	bufferTokens  chan []byte                  // control concurrency of multipart uploads
 }
 
@@ -251,7 +236,13 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 				fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err)
 			}
 		}
-		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Second)
+		retryAfterDuration := time.Duration(retryAfter) * time.Second
+		if f.pacer.GetSleep() < retryAfterDuration {
+			fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err)
+			// We set 1/2 the value here because the pacer will double it immediately
+			f.pacer.SetSleep(retryAfterDuration / 2)
+		}
+		return true, err
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -322,7 +313,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -357,7 +348,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		srv:    rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -952,13 +943,6 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
-		if apiErr, ok := err.(*api.Error); ok {
-			if apiErr.Code == "already_hidden" {
-				// sometimes eventual consistency causes this, so
-				// ignore this error since it is harmless
-				return nil
-			}
-		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1306,17 +1290,9 @@ var _ io.ReadCloser = &openFile{}
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	opts := rest.Opts{
 		Method:  "GET",
+		RootURL: o.fs.info.DownloadURL,
 		Options: options,
 	}
-
-	// Use downloadUrl from backblaze if downloadUrl is not set
-	// otherwise use the custom downloadUrl
-	if o.fs.opt.DownloadURL == "" {
-		opts.RootURL = o.fs.info.DownloadURL
-	} else {
-		opts.RootURL = o.fs.opt.DownloadURL
-	}
-
 	// Download by id if set otherwise by name
 	if o.id != "" {
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
@@ -1477,7 +1453,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// Content-Type b2/x-auto to automatically set the stored Content-Type
 	// post upload. In the case where a file extension is absent or the
 	// lookup fails, the Content-Type is set to application/octet-stream. The
-	// Content-Type mappings can be pursued here.
+	// Content-Type mappings can be purused here.
 	//
 	// X-Bz-Content-Sha1
 	// required
@@ -1524,6 +1500,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		},
 		ContentLength: &size,
 	}
+	// for go1.8 (see release notes) we must nil the Body if we want a
+	// "Content-Length: 0" header which b2 requires for all files.
+	if size == 0 {
+		opts.Body = nil
+	}
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {

backend/b2/upload.go

@@ -116,10 +116,8 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
 		},
 	}
 	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
-		}
+	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+		request.Info[sha1Key] = calculatedSha1
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {

backend/box/api/types.go

@@ -45,7 +45,7 @@ type Error struct {
 	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,7 +57,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo

backend/box/box.go

@@ -111,7 +111,7 @@ type Fs struct {
 	features     *fs.Features          // optional features
 	srv          *rest.Client          // the connection to the one drive server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *fs.Pacer             // pacer for API calls
+	pacer        *pacer.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
 }
@@ -171,13 +171,13 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
+	authRety := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
+		authRety = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // substitute reserved characters for box
@@ -260,7 +260,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:        root,
 		opt:         *opt,
 		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{
@@ -530,10 +530,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)

backend/box/upload.go

@@ -211,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		if reqSize >= int64(chunkSize) {
+			reqSize = int64(chunkSize)
 		}
 
 		// Make a block of memory

backend/cache/cache.go

@@ -576,7 +576,7 @@ The slice indices are similar to Python slices: start[:end]
 
 start is the 0 based chunk number from the beginning of the file
 to fetch inclusive. end is 0 based chunk number from the beginning
-of the file to fetch exclusive.
+of the file to fetch exclisive.
 Both values can be negative, in which case they count from the back
 of the file. The value "-5:" represents the last 5 chunks of a file.
 
@@ -870,7 +870,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 	}
 }
 
-// ChangeNotify can subscribe multiple callers
+// ChangeNotify can subsribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
@@ -1191,7 +1191,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1287,7 +1287,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1549,7 +1549,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantCopy
@@ -1625,7 +1625,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 
 	// if this is a temp object then we perform the changes locally
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantMove

backend/cache/cache_internal_test.go

@@ -387,10 +387,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), testSize)
+	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -726,7 +726,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// Call the rc function
 	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
-	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -736,21 +735,18 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
-	_, err = runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err := runInstance.list(t, rootFs, "")
 
 	// create some rand test data
 	testData2 := randStringBytes(int(chunkSize))
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
 
 	// list should have 1 item only
-	li1, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err = runInstance.list(t, rootFs, "")
 	require.Len(t, li1, 1)
 
 	// Call the rc function
 	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
-	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -758,7 +754,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// list should have 2 items now
 	li2, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
 	require.Len(t, li2, 2)
 }
 
@@ -1495,8 +1490,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 	var err error
 
 	if r.useMount {
-		var f *os.File
-		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
@@ -1506,8 +1500,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		}()
 		_, err = f.WriteString(data + append)
 	} else {
-		var obj1 fs.Object
-		obj1, err = rootFs.NewObject(src)
+		obj1, err := rootFs.NewObject(src)
 		if err != nil {
 			return err
 		}
@@ -1639,13 +1632,15 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	}
-	if f.Features().UnWrap != nil {
-		cfs, ok := f.Features().UnWrap().(*cache.Fs)
-		if ok {
-			return cfs, nil
+	} else {
+		if f.Features().UnWrap != nil {
+			cfs, ok := f.Features().UnWrap().(*cache.Fs)
+			if ok {
+				return cfs, nil
+			}
 		}
 	}
+
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/storage_persistent.go

@@ -398,7 +398,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), encoded)
+		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -809,7 +809,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), encoded)
+		err = bucket.Put([]byte(destPath), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -1023,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1049,7 +1049,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), encoded)
+			err = bucket.Put([]byte(destPath), []byte(encoded))
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}

backend/crypt/cipher.go

@@ -463,7 +463,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(newRune)
+			_, _ = result.WriteRune(rune(newRune))
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -748,7 +748,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retrieve the nonce
+	// retreive the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil

backend/crypt/crypt.go

@@ -122,7 +122,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -555,7 +555,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 }
 
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calculates the hash given by HashType on the fly
+// src with it, and calcuates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {

backend/drive/drive.go

@@ -21,7 +21,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -40,7 +39,6 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -56,8 +54,7 @@ const (
 	driveFolderType             = "application/vnd.google-apps.folder"
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
-	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
-	defaultBurst                = 100
+	minSleep                    = 10 * time.Millisecond
 	defaultExportExtensions     = "docx,xlsx,pptx,svg"
 	scopePrefix                 = "https://www.googleapis.com/auth/"
 	defaultScope                = "drive"
@@ -186,10 +183,10 @@ func init() {
 		},
 		Options: []fs.Option{{
 			Name: config.ConfigClientID,
-			Help: "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance.",
+			Help: "Google Application Client Id\nLeave blank normally.",
 		}, {
 			Name: config.ConfigClientSecret,
-			Help: "Google Application Client Secret\nSetting your own is recommended.",
+			Help: "Google Application Client Secret\nLeave blank normally.",
 		}, {
 			Name: "scope",
 			Help: "Scope that rclone should use when requesting access from drive.",
@@ -240,22 +237,6 @@ func init() {
 			Default:  false,
 			Help:     "Skip google documents in all listings.\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
-		}, {
-			Name:    "skip_checksum_gphotos",
-			Default: false,
-			Help: `Skip MD5 checksum on Google photos and videos only.
-
-Use this if you get checksum errors when transferring Google photos or
-videos.
-
-Setting this flag will cause Google photos and videos to return a
-blank MD5 checksum.
-
-Google photos are identifed by being in the "photos" space.
-
-Corrupted checksums are caused by Google modifying the image/video but
-not updating the checksum.`,
-			Advanced: true,
 		}, {
 			Name:    "shared_with_me",
 			Default: false,
@@ -375,16 +356,6 @@ will download it anyway.`,
 			Default:  fs.SizeSuffix(-1),
 			Help:     "If Object's are greater, use drive v2 API to download.",
 			Advanced: true,
-		}, {
-			Name:     "pacer_min_sleep",
-			Default:  defaultMinSleep,
-			Help:     "Minimum time to sleep between API calls.",
-			Advanced: true,
-		}, {
-			Name:     "pacer_burst",
-			Default:  defaultBurst,
-			Help:     "Number of API calls to allow without sleeping.",
-			Advanced: true,
 		}},
 	})
 
@@ -412,7 +383,6 @@ type Options struct {
 	AuthOwnerOnly             bool          `config:"auth_owner_only"`
 	UseTrash                  bool          `config:"use_trash"`
 	SkipGdocs                 bool          `config:"skip_gdocs"`
-	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool          `config:"shared_with_me"`
 	TrashedOnly               bool          `config:"trashed_only"`
 	Extensions                string        `config:"formats"`
@@ -428,8 +398,6 @@ type Options struct {
 	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
 	KeepRevisionForever       bool          `config:"keep_revision_forever"`
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
-	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
-	PacerBurst                int           `config:"pacer_burst"`
 }
 
 // Fs represents a remote drive server
@@ -443,7 +411,7 @@ type Fs struct {
 	client           *http.Client       // authorized client
 	rootFolderID     string             // the id of the root folder
 	dirCache         *dircache.DirCache // Map of directory path to directory id
-	pacer            *fs.Pacer          // To pace the API calls
+	pacer            *pacer.Pacer       // To pace the API calls
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
@@ -499,7 +467,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whether a given err rates being retried
+// shouldRetry determines whehter a given err rates being retried
 func shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
@@ -632,9 +600,6 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, inc
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
-	if f.opt.SkipChecksumGphotos {
-		fields += ",spaces"
-	}
 
 	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
 
@@ -696,33 +661,28 @@ func isPowerOfTwo(x int64) bool {
 }
 
 // add a charset parameter to all text/* MIME types
-func fixMimeType(mimeTypeIn string) string {
-	if mimeTypeIn == "" {
-		return ""
-	}
-	mediaType, param, err := mime.ParseMediaType(mimeTypeIn)
+func fixMimeType(mimeType string) string {
+	mediaType, param, err := mime.ParseMediaType(mimeType)
 	if err != nil {
-		return mimeTypeIn
+		return mimeType
 	}
-	mimeTypeOut := mimeTypeIn
-	if strings.HasPrefix(mediaType, "text/") && param["charset"] == "" {
+	if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
 		param["charset"] = "utf-8"
-		mimeTypeOut = mime.FormatMediaType(mediaType, param)
+		mimeType = mime.FormatMediaType(mediaType, param)
 	}
-	if mimeTypeOut == "" {
-		panic(errors.Errorf("unable to fix MIME type %q", mimeTypeIn))
-	}
-	return mimeTypeOut
+	return mimeType
 }
-func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
-	out = make(map[string][]string, len(in))
-	for k, v := range in {
+func fixMimeTypeMap(m map[string][]string) map[string][]string {
+	for _, v := range m {
 		for i, mt := range v {
-			v[i] = fixMimeType(mt)
+			fixed := fixMimeType(mt)
+			if fixed == "" {
+				panic(errors.Errorf("unable to fix MIME type %q", mt))
+			}
+			v[i] = fixed
 		}
-		out[fixMimeType(k)] = v
 	}
-	return out
+	return m
 }
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
@@ -784,7 +744,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 	listFailed := false
 	for {
 		var teamDrives *drive.TeamDriveList
-		err = newPacer(opt).Call(func() (bool, error) {
+		err = newPacer().Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Do()
 			return shouldRetry(err)
 		})
@@ -814,8 +774,8 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer(opt *Options) *fs.Pacer {
-	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
+func newPacer() *pacer.Pacer {
+	return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
@@ -888,7 +848,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -919,7 +879,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		pacer: newPacer(opt),
+		pacer: newPacer(),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
@@ -927,7 +887,6 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
-		ServerSideAcrossConfigs: true,
 	}).Fill(f)
 
 	// Create a new authorized Drive client.
@@ -1022,15 +981,6 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 
 // newRegularObject creates a fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
-	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
-	if f.opt.SkipChecksumGphotos {
-		for _, space := range info.Spaces {
-			if space == "photos" {
-				info.Md5Checksum = ""
-				break
-			}
-		}
-	}
 	return &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, info.Id),
@@ -1375,46 +1325,17 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-// listREntry is a task to be executed by a litRRunner
-type listREntry struct {
-	id, path string
-}
-
-// listRSlices is a helper struct to sort two slices at once
-type listRSlices struct {
-	dirs  []string
-	paths []string
-}
-
-func (s listRSlices) Sort() {
-	sort.Sort(s)
-}
-
-func (s listRSlices) Len() int {
-	return len(s.dirs)
-}
-
-func (s listRSlices) Swap(i, j int) {
-	s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
-	s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
-}
-
-func (s listRSlices) Less(i, j int) bool {
-	return s.dirs[i] < s.dirs[j]
-}
-
 // listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
 //
-// In each cycle it will read up to grouping entries from the in channel without blocking.
+// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
+func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
 	var dirs []string
-	var paths []string
 
 	for dir := range in {
-		dirs = append(dirs[:0], dir.id)
-		paths = append(paths[:0], dir.path)
+		dirs = append(dirs[:0], dir)
+		wait := time.After(10 * time.Millisecond)
 	waitloop:
 		for i := 1; i < grouping; i++ {
 			select {
@@ -1422,32 +1343,31 @@ func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- er
 				if !ok {
 					break waitloop
 				}
-				dirs = append(dirs, d.id)
-				paths = append(paths, d.path)
-			default:
+				dirs = append(dirs, d)
+			case <-wait:
+				break waitloop
 			}
 		}
-		listRSlices{dirs, paths}.Sort()
 		var iErr error
 		_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
-			for _, parent := range item.Parents {
-				// only handle parents that are in the requested dirs list
-				i := sort.SearchStrings(dirs, parent)
-				if i == len(dirs) || dirs[i] != parent {
-					continue
-				}
-				remote := path.Join(paths[i], item.Name)
-				entry, err := f.itemToDirEntry(remote, item)
-				if err != nil {
-					iErr = err
-					return true
+			parentPath := ""
+			if len(item.Parents) > 0 {
+				p, ok := f.dirCache.GetInv(item.Parents[0])
+				if ok {
+					parentPath = p
 				}
+			}
+			remote := path.Join(parentPath, item.Name)
+			entry, err := f.itemToDirEntry(remote, item)
+			if err != nil {
+				iErr = err
+				return true
+			}
 
-				err = cb(entry)
-				if err != nil {
-					iErr = err
-					return true
-				}
+			err = cb(entry)
+			if err != nil {
+				iErr = err
+				return true
 			}
 			return false
 		})
@@ -1498,44 +1418,30 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
-	if directoryID == "root" {
-		var info *drive.File
-		err = f.pacer.CallNoRetry(func() (bool, error) {
-			info, err = f.svc.Files.Get("root").
-				Fields("id").
-				SupportsTeamDrives(f.isTeamDrive).
-				Do()
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return err
-		}
-		directoryID = info.Id
-	}
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
-	in := make(chan listREntry, inputBuffer)
+	in := make(chan string, inputBuffer)
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
-	overflow := []listREntry{}
+	overfflow := []string{}
 
 	cb := func(entry fs.DirEntry) error {
 		mu.Lock()
 		defer mu.Unlock()
 		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
 			select {
-			case in <- listREntry{d.ID(), d.Remote()}:
+			case in <- d.ID():
 				wg.Add(1)
 			default:
-				overflow = append(overflow, listREntry{d.ID(), d.Remote()})
+				overfflow = append(overfflow, d.ID())
 			}
 		}
 		return list.Add(entry)
 	}
 
 	wg.Add(1)
-	in <- listREntry{directoryID, dir}
+	in <- directoryID
 
 	for i := 0; i < fs.Config.Checkers; i++ {
 		go f.listRRunner(&wg, in, out, cb, grouping)
@@ -1544,18 +1450,18 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 		// wait until the all directories are processed
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
-		for len(overflow) > 0 {
+		for len(overfflow) > 0 {
 			mu.Lock()
-			l := len(overflow)
-			// only fill half of the channel to prevent entries beeing put into overflow again
+			l := len(overfflow)
+			// only fill half of the channel to prevent entries beeing put into overfflow again
 			if l > inputBuffer/2 {
 				l = inputBuffer / 2
 			}
 			wg.Add(l)
-			for _, d := range overflow[:l] {
+			for _, d := range overfflow[:l] {
 				in <- d
 			}
-			overflow = overflow[l:]
+			overfflow = overfflow[l:]
 			mu.Unlock()
 
 			// wait again for the completion of all directories
@@ -1746,14 +1652,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
+				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.rmdir(srcDir.ID(), true)
 		if err != nil {
-			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -2172,7 +2078,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // ChangeNotify calls the passed function with a path that has had changes.
 // If the implementation uses polling, it should adhere to the given interval.
 //
-// Automatically restarts itself in case of unexpected behavior of the remote.
+// Automatically restarts itself in case of unexpected behaviour of the remote.
 //
 // Close the returned channel to stop being notified.
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
@@ -2279,13 +2185,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPage
 
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
-					for _, parent := range change.File.Parents {
-						if parentPath, ok := f.dirCache.GetInv(parent); ok {
-							// and append the drive file name to compute the full file name
-							newPath := path.Join(parentPath, change.File.Name)
-							// this will now clear the actual file too
-							pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
-						}
+					if parentPath, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
+						// and append the drive file name to compute the full file name
+						newPath := path.Join(parentPath, change.File.Name)
+						// this will now clear the actual file too
+						pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
 					}
 				} else { // a true root object that is changed
 					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
@@ -2465,10 +2369,6 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
 		return req, nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
-	if o.bytes == 0 {
-		// Don't supply range requests for 0 length objects as they always fail
-		delete(req.Header, "Range")
-	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {
@@ -2581,32 +2481,16 @@ func (o *documentObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err e
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
-	var offset, end int64 = 0, -1
-	var newOptions = options[:0]
 	for _, o := range options {
-		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
-		// So do a subset of them manually
-		switch x := o.(type) {
-		case *fs.RangeOption:
-			offset, end = x.Start, x.End
-		case *fs.SeekOption:
-			offset, end = x.Offset, -1
-		default:
-			newOptions = append(newOptions, o)
+		if _, ok := o.(*fs.RangeOption); ok {
+			return nil, errors.New("partial downloads are not supported while exporting Google Documents")
 		}
 	}
-	options = newOptions
-	if offset != 0 {
-		return nil, errors.New("partial downloads are not supported while exporting Google Documents")
-	}
 	in, err = o.baseObject.open(o.url, options...)
 	if in != nil {
 		in = &openDocumentFile{o: o, in: in}
 	}
-	if end >= 0 {
-		in = readers.NewLimitedReadCloser(in, end-offset+1)
-	}
 	return
 }
 func (o *linkObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
@@ -2672,9 +2556,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
-	if err != nil {
-		return err
-	}
 	switch newO := newO.(type) {
 	case *Object:
 		*o = *newO
@@ -2713,9 +2594,6 @@ func (o *documentObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.O
 	remote = remote[:len(remote)-o.extLen]
 
 	newO, err := o.fs.newObjectWithInfo(remote, info)
-	if err != nil {
-		return err
-	}
 	switch newO := newO.(type) {
 	case *documentObject:
 		*o = *newO

backend/drive/upload.go

@@ -185,7 +185,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionally on
+	// any other 2xx codes, so we parse it unconditionaly on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err

backend/dropbox/dropbox.go

@@ -130,8 +130,8 @@ Any files larger than this will be uploaded in chunks of this size.
 Note that chunks are buffered in memory (one at a time) so rclone can
 deal with retries.  Setting this larger will increase the speed
 slightly (at most 10%% for 128MB in tests) at the cost of using more
-memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
-			Default:  defaultChunkSize,
+memory.  It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}, {
 			Name:     "impersonate",
@@ -160,7 +160,7 @@ type Fs struct {
 	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *fs.Pacer      // To pace the API calls
+	pacer          *pacer.Pacer   // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 }
 
@@ -209,12 +209,12 @@ func shouldRetry(err error) (bool, error) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
 			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
-			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
+			time.Sleep(time.Duration(e.RateLimitError.RetryAfter) * time.Second)
 		}
 		return true, err
 	}
-	// Keep old behavior for backward compatibility
-	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") || baseErrString == "" {
+	// Keep old behaviour for backward compatibility
+	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
 		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
@@ -239,7 +239,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -273,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	config := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo

backend/ftp/ftp.go

@@ -15,7 +15,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -46,11 +45,6 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
-			}, {
-				Name:     "concurrency",
-				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-				Default:  0,
-				Advanced: true,
 			},
 		},
 	})
@@ -58,11 +52,10 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host        string `config:"host"`
-	User        string `config:"user"`
-	Pass        string `config:"pass"`
-	Port        string `config:"port"`
-	Concurrency int    `config:"concurrency"`
+	Host string `config:"host"`
+	User string `config:"user"`
+	Pass string `config:"pass"`
+	Port string `config:"port"`
 }
 
 // Fs represents a remote FTP server
@@ -77,7 +70,6 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
-	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -136,9 +128,6 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		f.tokens.Get()
-	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -158,9 +147,6 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		defer f.tokens.Put()
-	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -180,7 +166,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	f.poolMu.Unlock()
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
@@ -212,7 +198,6 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
-		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,

backend/googlecloudstorage/googlecloudstorage.go

@@ -16,7 +16,6 @@ FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 erro
 */
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -46,8 +45,6 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
-
-	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
 )
 
@@ -147,22 +144,6 @@ func init() {
 				Value: "publicReadWrite",
 				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",
 			}},
-		}, {
-			Name: "bucket_policy_only",
-			Help: `Access checks should use bucket-level IAM policies.
-
-If you want to upload objects to a bucket with Bucket Policy Only set
-then you will need to set this.
-
-When it is set, rclone:
-
-- ignores ACLs set on buckets
-- ignores ACLs set on objects
-- creates buckets with Bucket Policy Only set
-
-Docs: https://cloud.google.com/storage/docs/bucket-policy-only
-`,
-			Default: false,
 		}, {
 			Name: "location",
 			Help: "Location for the newly created buckets.",
@@ -181,36 +162,21 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "asia-east1",
 				Help:  "Taiwan.",
-			}, {
-				Value: "asia-east2",
-				Help:  "Hong Kong.",
 			}, {
 				Value: "asia-northeast1",
 				Help:  "Tokyo.",
-			}, {
-				Value: "asia-south1",
-				Help:  "Mumbai.",
 			}, {
 				Value: "asia-southeast1",
 				Help:  "Singapore.",
 			}, {
 				Value: "australia-southeast1",
 				Help:  "Sydney.",
-			}, {
-				Value: "europe-north1",
-				Help:  "Finland.",
 			}, {
 				Value: "europe-west1",
 				Help:  "Belgium.",
 			}, {
 				Value: "europe-west2",
 				Help:  "London.",
-			}, {
-				Value: "europe-west3",
-				Help:  "Frankfurt.",
-			}, {
-				Value: "europe-west4",
-				Help:  "Netherlands.",
 			}, {
 				Value: "us-central1",
 				Help:  "Iowa.",
@@ -223,9 +189,6 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "us-west1",
 				Help:  "Oregon.",
-			}, {
-				Value: "us-west2",
-				Help:  "California.",
 			}},
 		}, {
 			Name: "storage_class",
@@ -260,7 +223,6 @@ type Options struct {
 	ServiceAccountCredentials string `config:"service_account_credentials"`
 	ObjectACL                 string `config:"object_acl"`
 	BucketACL                 string `config:"bucket_acl"`
-	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
 	Location                  string `config:"location"`
 	StorageClass              string `config:"storage_class"`
 }
@@ -276,7 +238,7 @@ type Fs struct {
 	bucket     string           // the bucket we are working on
 	bucketOKMu sync.Mutex       // mutex to protect bucket OK
 	bucketOK   bool             // true if we have created the bucket
-	pacer      *fs.Pacer        // To pace the API calls
+	pacer      *pacer.Pacer     // To pace the API calls
 }
 
 // Object describes a storage object
@@ -320,7 +282,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whether a given err rates being retried
+// shouldRetry determines whehter a given err rates being retried
 func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
@@ -368,7 +330,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 
@@ -401,11 +363,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
 		if err != nil {
-			ctx := context.Background()
-			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
-			}
+			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
 		}
 	}
 
@@ -419,7 +377,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		opt:    *opt,
-		pacer:  fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -733,19 +691,8 @@ func (f *Fs) Mkdir(dir string) (err error) {
 		Location:     f.opt.Location,
 		StorageClass: f.opt.StorageClass,
 	}
-	if f.opt.BucketPolicyOnly {
-		bucket.IamConfiguration = &storage.BucketIamConfiguration{
-			BucketPolicyOnly: &storage.BucketIamConfigurationBucketPolicyOnly{
-				Enabled: true,
-			},
-		}
-	}
 	err = f.pacer.Call(func() (bool, error) {
-		insertBucket := f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket)
-		if !f.opt.BucketPolicyOnly {
-			insertBucket.PredefinedAcl(f.opt.BucketACL)
-		}
-		_, err = insertBucket.Do()
+		_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -1011,11 +958,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		insertObject := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
-		if !o.fs.opt.BucketPolicyOnly {
-			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
-		}
-		newObject, err = insertObject.Do()
+		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/http/http.go

@@ -6,7 +6,6 @@ package http
 
 import (
 	"io"
-	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -41,26 +40,7 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "https://example.com",
 				Help:  "Connect to example.com",
-			}, {
-				Value: "https://user:pass@example.com",
-				Help:  "Connect to example.com using a username and password",
 			}},
-		}, {
-			Name: "no_slash",
-			Help: `Set this if the site doesn't end directories with /
-
-Use this if your target website does not use / on the end of
-directories.
-
-A / on the end of a path is how rclone normally tells the difference
-between files and directories.  If this flag is set, then rclone will
-treat all files with Content-Type: text/html as directories and read
-URLs from them rather than downloading them.
-
-Note that this may cause rclone to confuse genuine HTML files with
-directories.`,
-			Default:  false,
-			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -69,7 +49,6 @@ directories.`,
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
-	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -269,7 +248,7 @@ func parseName(base *url.URL, name string) (string, error) {
 	}
 	// calculate the name relative to the base
 	name = u.Path[len(base.Path):]
-	// mustn't be empty
+	// musn't be empty
 	if name == "" {
 		return "", errNameIsEmpty
 	}
@@ -288,20 +267,14 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var (
-		walk func(*html.Node)
-		seen = make(map[string]struct{})
-	)
+	var walk func(*html.Node)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						if _, found := seen[name]; !found {
-							names = append(names, name)
-							seen[name] = struct{}{}
-						}
+						names = append(names, name)
 					}
 					break
 				}
@@ -326,16 +299,14 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil {
-		defer fs.CheckClose(res.Body, &err)
-		if res.StatusCode == http.StatusNotFound {
-			return nil, fs.ErrorDirNotFound
-		}
+	if err == nil && res.StatusCode == http.StatusNotFound {
+		return nil, fs.ErrorDirNotFound
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
+	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -379,16 +350,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			switch err = file.stat(); err {
-			case nil:
-				entries = append(entries, file)
-			case fs.ErrorNotAFile:
-				// ...found a directory not a file
-				dir := fs.NewDir(remote, timeUnset)
-				entries = append(entries, dir)
-			default:
+			if err = file.stat(); err != nil {
 				fs.Debugf(remote, "skipping because of error: %v", err)
+				continue
 			}
+			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -464,16 +430,6 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
-	// If NoSlash is set then check ContentType to see if it is a directory
-	if o.fs.opt.NoSlash {
-		mediaType, _, err := mime.ParseMediaType(o.contentType)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
-		}
-		if mediaType == "text/html" {
-			return fs.ErrorNotAFile
-		}
-	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -65,7 +65,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
+func testListRoot(t *testing.T, f fs.Fs) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -93,29 +93,15 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	if noSlash {
-		assert.Equal(t, int64(-1), e.Size())
-		_, ok = e.(fs.Directory)
-		assert.True(t, ok)
-	} else {
-		assert.Equal(t, int64(41), e.Size())
-		_, ok = e.(*Object)
-		assert.True(t, ok)
-	}
+	assert.Equal(t, int64(7), e.Size())
+	_, ok = e.(*Object)
+	assert.True(t, ok)
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f, false)
-}
-
-func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
-	f.(*Fs).opt.NoSlash = true
-	defer tidy()
-
-	testListRoot(t, f, true)
+	testListRoot(t, f)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -208,7 +194,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f, false)
+	testListRoot(t, f)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-<a href="two.html/file.txt">file.txt</a>
+potato

backend/hubic/hubic.go

@@ -9,10 +9,8 @@ package hubic
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/swift"
@@ -126,9 +124,7 @@ func (f *Fs) getCredentials() (err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
-		return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
+		return errors.Errorf("failed to get credentials: %s", resp.Status)
 	}
 	decoder := json.NewDecoder(resp.Body)
 	var result credentials

backend/jottacloud/api/types.go

@@ -9,10 +9,7 @@ import (
 )
 
 const (
-	// default time format for almost all request and responses
 	timeFormat = "2006-01-02-T15:04:05Z0700"
-	// the API server seems to use a different format
-	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
 )
 
 // Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -43,9 +40,6 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 // Return Time string in Jottacloud format
 func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 
-// APIString returns Time string in Jottacloud API format
-func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
-
 // Flag is a hacky type for checking if an attribute is present
 type Flag bool
 
@@ -64,15 +58,6 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	return attr, errors.New("unimplemented")
 }
 
-// TokenJSON is the struct representing the HTTP response from OAuth2
-// providers returning a token in JSON form.
-type TokenJSON struct {
-	AccessToken  string `json:"access_token"`
-	TokenType    string `json:"token_type"`
-	RefreshToken string `json:"refresh_token"`
-	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
-}
-
 /*
 GET http://www.jottacloud.com/JFS/<account>
 
@@ -280,37 +265,3 @@ func (e *Error) Error() string {
 	}
 	return out
 }
-
-// AllocateFileRequest to prepare an upload to Jottacloud
-type AllocateFileRequest struct {
-	Bytes    int64  `json:"bytes"`
-	Created  string `json:"created"`
-	Md5      string `json:"md5"`
-	Modified string `json:"modified"`
-	Path     string `json:"path"`
-}
-
-// AllocateFileResponse for upload requests
-type AllocateFileResponse struct {
-	Name      string `json:"name"`
-	Path      string `json:"path"`
-	State     string `json:"state"`
-	UploadID  string `json:"upload_id"`
-	UploadURL string `json:"upload_url"`
-	Bytes     int64  `json:"bytes"`
-	ResumePos int64  `json:"resume_pos"`
-}
-
-// UploadResponse after an upload
-type UploadResponse struct {
-	Name      string      `json:"name"`
-	Path      string      `json:"path"`
-	Kind      string      `json:"kind"`
-	ContentID string      `json:"content_id"`
-	Bytes     int64       `json:"bytes"`
-	Md5       string      `json:"md5"`
-	Created   int64       `json:"created"`
-	Modified  int64       `json:"modified"`
-	Deleted   interface{} `json:"deleted"`
-	Mime      string      `json:"mime"`
-}

backend/jottacloud/jottacloud.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -27,41 +26,22 @@ import (
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
-	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
-	"golang.org/x/oauth2"
 )
 
 // Globals
 const (
-	minSleep                    = 10 * time.Millisecond
-	maxSleep                    = 2 * time.Second
-	decayConstant               = 2 // bigger for slower decay, exponential
-	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Sync" // nolint
-	rootURL                     = "https://www.jottacloud.com/jfs/"
-	apiURL                      = "https://api.jottacloud.com/files/v1/"
-	baseURL                     = "https://www.jottacloud.com/"
-	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
-	cachePrefix                 = "rclone-jcmd5-"
-	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
-	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
-	configUsername              = "user"
-)
-
-var (
-	// Description of how to auth for this app for a personal account
-	oauthConfig = &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
-		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
-	}
+	minSleep          = 10 * time.Millisecond
+	maxSleep          = 2 * time.Second
+	decayConstant     = 2 // bigger for slower decay, exponential
+	defaultDevice     = "Jotta"
+	defaultMountpoint = "Sync"
+	rootURL           = "https://www.jottacloud.com/jfs/"
+	apiURL            = "https://api.jottacloud.com"
+	shareURL          = "https://www.jottacloud.com/"
+	cachePrefix       = "rclone-jcmd5-"
 )
 
 // Register with Fs
@@ -70,71 +50,13 @@ func init() {
 		Name:        "jottacloud",
 		Description: "JottaCloud",
 		NewFs:       NewFs,
-		Config: func(name string, m configmap.Mapper) {
-			tokenString, ok := m.Get("token")
-			if ok && tokenString != "" {
-				fmt.Printf("Already have a token - refresh?\n")
-				if !config.Confirm() {
-					return
-				}
-			}
-
-			username, ok := m.Get(configUsername)
-			if !ok {
-				log.Fatalf("No username defined")
-			}
-			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
-
-			// prepare out token request with username and password
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
-			values := url.Values{}
-			values.Set("grant_type", "PASSWORD")
-			values.Set("password", password)
-			values.Set("username", username)
-			values.Set("client_id", oauthConfig.ClientID)
-			values.Set("client_secret", oauthConfig.ClientSecret)
-			opts := rest.Opts{
-				Method:      "POST",
-				RootURL:     oauthConfig.Endpoint.AuthURL,
-				ContentType: "application/x-www-form-urlencoded",
-				Parameters:  values,
-			}
-
-			var jsonToken api.TokenJSON
-			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
-			if err != nil {
-				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
-				if resp != nil {
-					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
-						fmt.Printf("Enter verification code> ")
-						authCode := config.ReadLine()
-						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
-						opts.ExtraHeaders = make(map[string]string)
-						opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-						resp, err = srv.CallJSON(&opts, nil, &jsonToken)
-					}
-				}
-				if err != nil {
-					log.Fatalf("Failed to get resource token: %v", err)
-				}
-			}
-
-			var token oauth2.Token
-			token.AccessToken = jsonToken.AccessToken
-			token.RefreshToken = jsonToken.RefreshToken
-			token.TokenType = jsonToken.TokenType
-			token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
-
-			// finally save them in the config
-			err = oauthutil.PutToken(name, m, &token, true)
-			if err != nil {
-				log.Fatalf("Error while setting token: %s", err)
-			}
-		},
 		Options: []fs.Option{{
-			Name: configUsername,
-			Help: "User Name:",
+			Name: "user",
+			Help: "User Name",
+		}, {
+			Name:       "pass",
+			Help:       "Password.",
+			IsPassword: true,
 		}, {
 			Name:     "mountpoint",
 			Help:     "The mountpoint to use.",
@@ -161,11 +83,6 @@ func init() {
 			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:     "upload_resume_limit",
-			Help:     "Files bigger than this can be resumed if the upload fail's.",
-			Default:  fs.SizeSuffix(10 * 1024 * 1024),
-			Advanced: true,
 		}},
 	})
 }
@@ -173,25 +90,23 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
+	Pass               string        `config:"pass"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
 	Unlink             bool          `config:"unlink"`
-	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
 }
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name         string
-	root         string
-	user         string
-	opt          Options
-	features     *fs.Features
-	endpointURL  string
-	srv          *rest.Client
-	apiSrv       *rest.Client
-	pacer        *fs.Pacer
-	tokenRenewer *oauthutil.Renew // renew the token on expiry
+	name        string
+	root        string
+	user        string
+	opt         Options
+	features    *fs.Features
+	endpointURL string
+	srv         *rest.Client
+	pacer       *pacer.Pacer
 }
 
 // Object describes a jottacloud object
@@ -346,29 +261,6 @@ func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
 
-// Jottacloud requires the grant_type 'refresh_token' string
-// to be uppercase and throws a 400 Bad Request if we use the
-// lower case used by the oauth2 module
-//
-// This filter catches all refresh requests, reads the body,
-// changes the case and then sends it on
-func grantTypeFilter(req *http.Request) {
-	if tokenURL == req.URL.String() {
-		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
-		if err != nil {
-			return
-		}
-		_ = req.Body.Close()
-
-		// make the refresh token upper case
-		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
-
-		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
-	}
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -381,32 +273,25 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	// add jottacloud to the long list of sites that don't follow the oauth spec correctly
-	oauth2.RegisterBrokenAuthHeaderProvider("https://www.jottacloud.com/")
+	user := config.FileGet(name, "user")
+	pass := config.FileGet(name, "pass")
 
-	// the oauth client for the api servers needs
-	// a filter to fix the grant_type issues (see above)
-	baseClient := fshttp.NewClient(fs.Config)
-	if do, ok := baseClient.Transport.(interface {
-		SetRequestFilter(f func(req *http.Request))
-	}); ok {
-		do.SetRequestFilter(grantTypeFilter)
-	} else {
-		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
-	}
-	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
+	if opt.Pass != "" {
+		var err error
+		opt.Pass, err = obscure.Reveal(opt.Pass)
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't decrypt password")
+		}
 	}
 
 	f := &Fs{
-		name:   name,
-		root:   root,
-		user:   opt.User,
-		opt:    *opt,
-		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
-		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		name: name,
+		root: root,
+		user: opt.User,
+		opt:  *opt,
+		//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
+		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -414,13 +299,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 	}).Fill(f)
-	f.srv.SetErrorHandler(errorHandler)
 
-	// Renew the token in the background
-	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.readMetaDataForPath("")
-		return err
-	})
+	if user == "" || pass == "" {
+		return nil, errors.New("jottacloud needs user and password")
+	}
+
+	f.srv.SetUserPass(opt.User, opt.Pass)
+	f.srv.SetErrorHandler(errorHandler)
 
 	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
@@ -446,6 +331,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+
 	return f, nil
 }
 
@@ -462,7 +348,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData(false) // reads info and meta, returning an error
+		err = o.readMetaData() // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -510,7 +396,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	//fmt.Printf("List: %s\n", f.filePath(dir))
+	//fmt.Printf("List: %s\n", dir)
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   f.filePath(dir),
@@ -772,7 +658,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copies or moves directories or files depending on the method parameter
+// copyOrMoves copys or moves directories or files depending on the mthod parameter
 func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -790,6 +676,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
 	if err != nil {
 		return nil, err
 	}
+
 	return info, nil
 }
 
@@ -937,7 +824,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if result.PublicSharePath == "" {
 		return "", errors.New("couldn't create public link - no link path received")
 	}
-	link = path.Join(baseURL, result.PublicSharePath)
+	link = path.Join(shareURL, result.PublicSharePath)
 	return link, nil
 }
 
@@ -993,7 +880,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return 0
@@ -1009,24 +896,21 @@ func (o *Object) MimeType() string {
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.hasMetaData = true
-	o.size = info.Size
+	o.size = int64(info.Size)
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
 	o.modTime = time.Time(info.ModifiedAt)
 	return nil
 }
 
-func (o *Object) readMetaData(force bool) (err error) {
-	if o.hasMetaData && !force {
+func (o *Object) readMetaData() (err error) {
+	if o.hasMetaData {
 		return nil
 	}
 	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		return err
 	}
-	if info.Deleted {
-		return fs.ErrorObjectNotFound
-	}
 	return o.setMetaData(info)
 }
 
@@ -1035,7 +919,7 @@ func (o *Object) readMetaData(force bool) (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1083,7 +967,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
 	// we need a MD5
 	md5Hasher := md5.New()
-	// use the teeReader to write to the local file AND calculate the MD5 while doing so
+	// use the teeReader to write to the local file AND caclulate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
 
 	// nothing to clean up by default
@@ -1156,74 +1040,43 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		in = wrap(in)
 	}
 
-	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
+	var result api.JottaFile
 	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "allocate",
-		ExtraHeaders: make(map[string]string),
-	}
-	fileDate := api.Time(src.ModTime()).APIString()
-
-	// the allocate request
-	var request = api.AllocateFileRequest{
-		Bytes:    size,
-		Created:  fileDate,
-		Modified: fileDate,
-		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
+		Method:        "POST",
+		Path:          o.filePath(),
+		Body:          in,
+		ContentType:   fs.MimeType(src),
+		ContentLength: &size,
+		ExtraHeaders:  make(map[string]string),
+		Parameters:    url.Values{},
 	}
 
-	// send it
-	var response api.AllocateFileResponse
+	opts.ExtraHeaders["JMd5"] = md5String
+	opts.Parameters.Set("cphash", md5String)
+	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
+	// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
+	opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
+
+	// Parameters observed in other implementations
+	//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
+	//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
+	//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
+	//opts.ExtraHeaders["jx_csid"] = ""
+	//opts.ExtraHeaders["jx_lisence"] = ""
+
+	opts.Parameters.Set("umode", "nomultipart")
+
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
+		resp, err = o.fs.srv.CallXML(&opts, nil, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
-	if response.State != "COMPLETED" {
-		// how much do we still have to upload?
-		remainingBytes := size - response.ResumePos
-		opts = rest.Opts{
-			Method:        "POST",
-			RootURL:       response.UploadURL,
-			ContentLength: &remainingBytes,
-			ContentType:   "application/octet-stream",
-			Body:          in,
-			ExtraHeaders:  make(map[string]string),
-		}
-		if response.ResumePos != 0 {
-			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
-		}
-
-		// copy the already uploaded bytes into the trash :)
-		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
-		if err != nil {
-			return err
-		}
-
-		// send the remaining bytes
-		resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
-		if err != nil {
-			return err
-		}
-
-		// finally update the meta data
-		o.hasMetaData = true
-		o.size = result.Bytes
-		o.md5 = result.Md5
-		o.modTime = time.Unix(result.Modified/1000, 0)
-	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
-		return o.readMetaData(true)
-	}
-
-	return nil
+	// TODO: Check returned Metadata? Timeout on big uploads?
+	return o.setMetaData(&result)
 }
 
 // Remove an object

backend/jottacloud/replace.go

@@ -2,7 +2,7 @@
 Translate file names for JottaCloud adapted from OneDrive
 
 
-The following characters are JottaCloud reserved characters, and can't
+The following characters are JottaClous reserved characters, and can't
 be used in JottaCloud folder and file names.
 
   jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"

backend/koofr/koofr.go

@@ -1,589 +0,0 @@
-package koofr
-
-import (
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
-	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fs/hash"
-
-	httpclient "github.com/koofr/go-httpclient"
-	koofrclient "github.com/koofr/go-koofrclient"
-)
-
-// Register Fs with rclone
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "koofr",
-		Description: "Koofr",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "The Koofr API endpoint to use",
-				Default:  "https://app.koofr.net",
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "mountid",
-				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
-				Required: false,
-				Default:  "",
-				Advanced: true,
-			}, {
-				Name:     "user",
-				Help:     "Your Koofr user name",
-				Required: true,
-			}, {
-				Name:       "password",
-				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
-				IsPassword: true,
-				Required:   true,
-			},
-		},
-	})
-}
-
-// Options represent the configuration of the Koofr backend
-type Options struct {
-	Endpoint string `config:"endpoint"`
-	MountID  string `config:"mountid"`
-	User     string `config:"user"`
-	Password string `config:"password"`
-}
-
-// A Fs is a representation of a remote Koofr Fs
-type Fs struct {
-	name     string
-	mountID  string
-	root     string
-	opt      Options
-	features *fs.Features
-	client   *koofrclient.KoofrClient
-}
-
-// An Object on the remote Koofr Fs
-type Object struct {
-	fs     *Fs
-	remote string
-	info   koofrclient.FileInfo
-}
-
-func base(pth string) string {
-	rv := path.Base(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-func dir(pth string) string {
-	rv := path.Dir(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-// String returns a string representation of the remote Object
-func (o *Object) String() string {
-	return o.remote
-}
-
-// Remote returns the remote path of the Object, relative to Fs root
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the Object
-func (o *Object) ModTime() time.Time {
-	return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000)
-}
-
-// Size return the size of the Object in bytes
-func (o *Object) Size() int64 {
-	return o.info.Size
-}
-
-// Fs returns a reference to the Koofr Fs containing the Object
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns an MD5 hash of the Object
-func (o *Object) Hash(typ hash.Type) (string, error) {
-	if typ == hash.MD5 {
-		return o.info.Hash, nil
-	}
-	return "", nil
-}
-
-// fullPath returns full path of the remote Object (including Fs root)
-func (o *Object) fullPath() string {
-	return o.fs.fullPath(o.remote)
-}
-
-// Storable returns true if the Object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime is not supported
-func (o *Object) SetModTime(mtime time.Time) error {
-	return nil
-}
-
-// Open opens the Object for reading
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
-	var sOff, eOff int64 = 0, -1
-
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.SeekOption:
-			sOff = x.Offset
-		case *fs.RangeOption:
-			sOff = x.Start
-			eOff = x.End
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if sOff == 0 && eOff < 0 {
-		return o.fs.client.FilesGet(o.fs.mountID, o.fullPath())
-	}
-	if sOff < 0 {
-		sOff = o.Size() - eOff
-		eOff = o.Size()
-	}
-	if eOff > o.Size() {
-		eOff = o.Size()
-	}
-	span := &koofrclient.FileSpan{
-		Start: sOff,
-		End:   eOff,
-	}
-	return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span)
-}
-
-// Update updates the Object contents
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := o.fullPath()
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err := o.fs.mkdir(dirPath)
-	if err != nil {
-		return err
-	}
-	info, err := o.fs.client.FilesPutOptions(o.fs.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return err
-	}
-	o.info = *info
-	return nil
-}
-
-// Remove deletes the remote Object
-func (o *Object) Remove() error {
-	return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath())
-}
-
-// Name returns the name of the Fs
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root returns the root path of the Fs
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a string representation of the Fs
-func (f *Fs) String() string {
-	return "koofr:" + f.mountID + ":" + f.root
-}
-
-// Features returns the optional features supported by this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Precision denotes that setting modification times is not supported
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns a set of hashes are Provided by the Fs
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// fullPath constructs a full, absolute path from a Fs root relative path,
-func (f *Fs) fullPath(part string) string {
-	return path.Join("/", f.root, part)
-}
-
-// NewFs constructs a new filesystem given a root path and configuration options
-func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
-	opt := new(Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-	pass, err := obscure.Reveal(opt.Password)
-	if err != nil {
-		return nil, err
-	}
-	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
-	basicAuth := fmt.Sprintf("Basic %s",
-		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
-	client.HTTPClient.Headers.Set("Authorization", basicAuth)
-	mounts, err := client.Mounts()
-	if err != nil {
-		return nil, err
-	}
-	f := &Fs{
-		name:   name,
-		root:   root,
-		opt:    *opt,
-		client: client,
-	}
-	f.features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          false,
-		BucketBased:             false,
-		CanHaveEmptyDirectories: true,
-	}).Fill(f)
-	for _, m := range mounts {
-		if opt.MountID != "" {
-			if m.Id == opt.MountID {
-				f.mountID = m.Id
-				break
-			}
-		} else if m.IsPrimary {
-			f.mountID = m.Id
-			break
-		}
-	}
-	if f.mountID == "" {
-		if opt.MountID == "" {
-			return nil, errors.New("Failed to find primary mount")
-		}
-		return nil, errors.New("Failed to find mount " + opt.MountID)
-	}
-	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
-	if err == nil && rootFile.Type != "dir" {
-		f.root = dir(f.root)
-		err = fs.ErrorIsFile
-	} else {
-		err = nil
-	}
-	return f, err
-}
-
-// List returns a list of items in a directory
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return nil, translateErrorsDir(err)
-	}
-	entries = make([]fs.DirEntry, len(files))
-	for i, file := range files {
-		if file.Type == "dir" {
-			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
-		} else {
-			entries[i] = &Object{
-				fs:     f,
-				info:   file,
-				remote: path.Join(dir, file.Name),
-			}
-		}
-	}
-	return entries, nil
-}
-
-// NewObject creates a new remote Object for a given remote path
-func (f *Fs) NewObject(remote string) (obj fs.Object, err error) {
-	info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote))
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	if info.Type == "dir" {
-		return nil, fs.ErrorNotAFile
-	}
-	return &Object{
-		fs:     f,
-		info:   info,
-		remote: remote,
-	}, nil
-}
-
-// Put updates a remote Object
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := f.fullPath(src.Remote())
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err = f.mkdir(dirPath)
-	if err != nil {
-		return nil, err
-	}
-	info, err := f.client.FilesPutOptions(f.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	return &Object{
-		fs:     f,
-		info:   *info,
-		remote: src.Remote(),
-	}, nil
-}
-
-// PutStream updates a remote Object with a stream of unknown size
-func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(in, src, options...)
-}
-
-// isBadRequest is a predicate which holds true iff the error returned was
-// HTTP status 400
-func isBadRequest(err error) bool {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusBadRequest {
-			return true
-		}
-	}
-	return false
-}
-
-// translateErrorsDir translates koofr errors to rclone errors (for a dir
-// operation)
-func translateErrorsDir(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorDirNotFound
-		}
-	}
-	return err
-}
-
-// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
-func translateErrorsObject(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorObjectNotFound
-		}
-	}
-	return err
-}
-
-// mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
-func (f *Fs) mkdir(fullPath string) error {
-	if fullPath == "/" {
-		return nil
-	}
-	info, err := f.client.FilesInfo(f.mountID, fullPath)
-	if err == nil && info.Type == "dir" {
-		return nil
-	}
-	err = translateErrorsDir(err)
-	if err != nil && err != fs.ErrorDirNotFound {
-		return err
-	}
-	dirs := strings.Split(fullPath, "/")
-	parent := "/"
-	for _, part := range dirs {
-		if part == "" {
-			continue
-		}
-		info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part))
-		if err != nil || info.Type != "dir" {
-			err = translateErrorsDir(err)
-			if err != nil && err != fs.ErrorDirNotFound {
-				return err
-			}
-			err = f.client.FilesNewFolder(f.mountID, parent, part)
-			if err != nil && !isBadRequest(err) {
-				return err
-			}
-		}
-		parent = path.Join(parent, part)
-	}
-	return nil
-}
-
-// Mkdir creates a directory at the given remote path. Creates ancestors if
-// necessary
-func (f *Fs) Mkdir(dir string) error {
-	fullPath := f.fullPath(dir)
-	return f.mkdir(fullPath)
-}
-
-// Rmdir removes an (empty) directory at the given remote path
-func (f *Fs) Rmdir(dir string) error {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	if len(files) > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-	err = f.client.FilesDelete(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	return nil
-}
-
-// Copy copies a remote Object to the given path
-func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	err = f.client.FilesCopy((src.(*Object)).fs.mountID,
-		(src.(*Object)).fs.fullPath((src.(*Object)).remote),
-		f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	return f.NewObject(remote)
-}
-
-// Move moves a remote Object to the given path
-func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
-	srcObj := src.(*Object)
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	err = f.client.FilesMove(srcObj.fs.mountID,
-		srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	return f.NewObject(remote)
-}
-
-// DirMove moves a remote directory to the given path
-func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
-	srcFs := src.(*Fs)
-	srcFullPath := srcFs.fullPath(srcRemote)
-	dstFullPath := f.fullPath(dstRemote)
-	if srcFs.mountID == f.mountID && srcFullPath == dstFullPath {
-		return fs.ErrorDirExists
-	}
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	return nil
-}
-
-// About reports space usage (with a MB precision)
-func (f *Fs) About() (*fs.Usage, error) {
-	mount, err := f.client.MountsDetails(f.mountID)
-	if err != nil {
-		return nil, err
-	}
-	return &fs.Usage{
-		Total:   fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024),
-		Used:    fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024),
-		Trashed: nil,
-		Other:   nil,
-		Free:    fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024),
-		Objects: nil,
-	}, nil
-}
-
-// Purge purges the complete Fs
-func (f *Fs) Purge() error {
-	err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath("")))
-	return err
-}
-
-// linkCreate is a Koofr API request for creating a public link
-type linkCreate struct {
-	Path string `json:"path"`
-}
-
-// link is a Koofr API response to creating a public link
-type link struct {
-	ID               string `json:"id"`
-	Name             string `json:"name"`
-	Path             string `json:"path"`
-	Counter          int64  `json:"counter"`
-	URL              string `json:"url"`
-	ShortURL         string `json:"shortUrl"`
-	Hash             string `json:"hash"`
-	Host             string `json:"host"`
-	HasPassword      bool   `json:"hasPassword"`
-	Password         string `json:"password"`
-	ValidFrom        int64  `json:"validFrom"`
-	ValidTo          int64  `json:"validTo"`
-	PasswordRequired bool   `json:"passwordRequired"`
-}
-
-// createLink makes a Koofr API call to create a public link
-func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) {
-	linkCreate := linkCreate{
-		Path: path,
-	}
-	linkData := link{}
-
-	request := httpclient.RequestData{
-		Method:         "POST",
-		Path:           "/api/v2/mounts/" + mountID + "/links",
-		ExpectedStatus: []int{http.StatusOK, http.StatusCreated},
-		ReqEncoding:    httpclient.EncodingJSON,
-		ReqValue:       linkCreate,
-		RespEncoding:   httpclient.EncodingJSON,
-		RespValue:      &linkData,
-	}
-
-	_, err := c.Request(&request)
-	if err != nil {
-		return nil, err
-	}
-	return &linkData, nil
-}
-
-// PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(remote string) (string, error) {
-	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
-	if err != nil {
-		return "", translateErrorsDir(err)
-	}
-	return linkData.ShortURL, nil
-}

backend/koofr/koofr_test.go

@@ -1,14 +0,0 @@
-package koofr_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestKoofr:",
-	})
-}

backend/local/about_unix.go

@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
-	bs := int64(s.Bsize) // nolint: unconvert
+	bs := int64(s.Bsize)
 	usage := &fs.Usage{
 		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
 		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use

backend/local/lchtimes.go

@@ -1,20 +0,0 @@
-// +build windows plan9
-
-package local
-
-import (
-	"time"
-)
-
-const haveLChtimes = false
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	// Does nothing
-	return nil
-}

backend/local/lchtimes_unix.go

@@ -1,28 +0,0 @@
-// +build !windows,!plan9
-
-package local
-
-import (
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const haveLChtimes = true
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	var utimes [2]unix.Timespec
-	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
-	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
-	if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
-		return &os.PathError{Op: "lchtimes", Path: name, Err: e}
-	}
-	return nil
-}

backend/local/local.go

@@ -2,10 +2,10 @@
 package local
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -29,7 +29,6 @@ import (
 
 // Constants
 const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
-const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
 
 // Register with Fs
 func init() {
@@ -51,13 +50,6 @@ func init() {
 			NoPrefix: true,
 			ShortOpt: "L",
 			Advanced: true,
-		}, {
-			Name:     "links",
-			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
-			Default:  false,
-			NoPrefix: true,
-			ShortOpt: "l",
-			Advanced: true,
 		}, {
 			Name: "skip_links",
 			Help: `Don't warn about skipped symlinks.
@@ -102,13 +94,12 @@ check can be disabled with this flag.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks    bool `config:"copy_links"`
-	TranslateSymlinks bool `config:"links"`
-	SkipSymlinks      bool `config:"skip_links"`
-	NoUTFNorm         bool `config:"no_unicode_normalization"`
-	NoCheckUpdated    bool `config:"no_check_updated"`
-	NoUNC             bool `config:"nounc"`
-	OneFileSystem     bool `config:"one_file_system"`
+	FollowSymlinks bool `config:"copy_links"`
+	SkipSymlinks   bool `config:"skip_links"`
+	NoUTFNorm      bool `config:"no_unicode_normalization"`
+	NoCheckUpdated bool `config:"no_check_updated"`
+	NoUNC          bool `config:"nounc"`
+	OneFileSystem  bool `config:"one_file_system"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -130,20 +121,17 @@ type Fs struct {
 
 // Object represents a local filesystem object
 type Object struct {
-	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path - properly UTF-8 encoded - for rclone
-	path           string // The local path - may not be properly UTF-8 encoded - for OS
-	size           int64  // file metadata - always present
-	mode           os.FileMode
-	modTime        time.Time
-	hashes         map[hash.Type]string // Hashes
-	translatedLink bool                 // Is this object a translated link
+	fs      *Fs    // The Fs this object is part of
+	remote  string // The remote path - properly UTF-8 encoded - for rclone
+	path    string // The local path - may not be properly UTF-8 encoded - for OS
+	size    int64  // file metadata - always present
+	mode    os.FileMode
+	modTime time.Time
+	hashes  map[hash.Type]string // Hashes
 }
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-
 // NewFs constructs an Fs from the path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -152,9 +140,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	if opt.TranslateSymlinks && opt.FollowSymlinks {
-		return nil, errLinksAndCopyLinks
-	}
 
 	if opt.NoUTFNorm {
 		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -182,7 +167,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	if err == nil && f.isRegular(fi.Mode()) {
+	if err == nil && fi.Mode().IsRegular() {
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -191,20 +176,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
-// Determine whether a file is a 'regular' file,
-// Symlinks are regular files, only if the TranslateSymlink
-// option is in-effect
-func (f *Fs) isRegular(mode os.FileMode) bool {
-	if !f.opt.TranslateSymlinks {
-		return mode.IsRegular()
-	}
-
-	// fi.Mode().IsRegular() tests that all mode bits are zero
-	// Since symlinks are accepted, test that all other bits are zero,
-	// except the symlink bit
-	return mode&os.ModeType&^os.ModeSymlink == 0
-}
-
 // Name of the remote (as passed into NewFs)
 func (f *Fs) Name() string {
 	return f.name
@@ -225,48 +196,28 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// caseInsensitive returns whether the remote is case insensitive or not
+// caseInsenstive returns whether the remote is case insensitive or not
 func (f *Fs) caseInsensitive() bool {
 	// FIXME not entirely accurate since you can have case
-	// sensitive Fses on darwin and case insensitive Fses on linux.
+	// sensitive Fses on darwin and case insenstive Fses on linux.
 	// Should probably check but that would involve creating a
 	// file in the remote to be most accurate which probably isn't
 	// desirable.
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
-// translateLink checks whether the remote is a translated link
-// and returns a new path, removing the suffix as needed,
-// It also returns whether this is a translated link at all
-//
-// for regular files, dstPath is returned unchanged
-func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
-	return newDstPath, isTranslatedLink
-}
-
 // newObject makes a half completed Object
 //
 // if dstPath is empty then it is made from remote
 func (f *Fs) newObject(remote, dstPath string) *Object {
-	translatedLink := false
-
 	if dstPath == "" {
 		dstPath = f.cleanPath(filepath.Join(f.root, remote))
 	}
 	remote = f.cleanRemote(remote)
-
-	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for dstPath
-		dstPath, translatedLink = translateLink(remote, dstPath)
-	}
-
 	return &Object{
-		fs:             f,
-		remote:         remote,
-		path:           dstPath,
-		translatedLink: translatedLink,
+		fs:     f,
+		remote: remote,
+		path:   dstPath,
 	}
 }
 
@@ -288,11 +239,6 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
-		// Handle the odd case, that a symlink was specified by name without the link suffix
-		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
-			return nil, fs.ErrorObjectNotFound
-		}
-
 	}
 	if o.mode.IsDir() {
 		return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -316,7 +262,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-
 	dir = f.dirNames.Load(dir)
 	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
 	remote := f.cleanRemote(dir)
@@ -373,10 +318,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 					entries = append(entries, d)
 				}
 			} else {
-				// Check whether this link should be translated
-				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += linkSuffix
-				}
 				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
 				if err != nil {
 					return nil, err
@@ -590,7 +531,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.fs.isRegular(dstObj.mode) {
+	} else if !dstObj.mode.IsRegular() {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -712,13 +653,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		var in io.ReadCloser
-
-		if !o.translatedLink {
-			in, err = file.Open(o.path)
-		} else {
-			in, err = o.openTranslatedLink(0, -1)
-		}
+		in, err := file.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -749,12 +684,7 @@ func (o *Object) ModTime() time.Time {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	var err error
-	if o.translatedLink {
-		err = lChtimes(o.path, modTime, modTime)
-	} else {
-		err = os.Chtimes(o.path, modTime, modTime)
-	}
+	err := os.Chtimes(o.path, modTime, modTime)
 	if err != nil {
 		return err
 	}
@@ -772,7 +702,7 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
+	if mode&os.ModeSymlink != 0 {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}
@@ -813,6 +743,7 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 	}
 
 	n, err = file.in.Read(p)
+	log.Printf("*** Read result n=%d, err=%v, len(p)=%d", n, err, len(p))
 	if n > 0 {
 		// Hash routines never return an error
 		_, _ = file.hash.Write(p[:n])
@@ -833,16 +764,6 @@ func (file *localOpenFile) Close() (err error) {
 	return err
 }
 
-// Returns a ReadCloser() object that contains the contents of a symbolic link
-func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
-	// Read the link and return the destination  it as the contents of the object
-	linkdst, err := os.Readlink(o.path)
-	if err != nil {
-		return nil, err
-	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
-}
-
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
@@ -862,11 +783,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
-	// Handle a translated link
-	if o.translatedLink {
-		return o.openTranslatedLink(offset, limit)
-	}
-
 	fd, err := file.Open(o.path)
 	if err != nil {
 		return
@@ -898,19 +814,8 @@ func (o *Object) mkdirAll() error {
 	return os.MkdirAll(dir, 0777)
 }
 
-type nopWriterCloser struct {
-	*bytes.Buffer
-}
-
-func (nwc nopWriterCloser) Close() error {
-	// noop
-	return nil
-}
-
 // Update the object from in with modTime and size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	var out io.WriteCloser
-
 	hashes := hash.Supported
 	for _, option := range options {
 		switch x := option.(type) {
@@ -924,23 +829,15 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	var symlinkData bytes.Buffer
-	// If the object is a regular file, create it.
-	// If it is a translated link, just read in the contents, and
-	// then create a symlink
-	if !o.translatedLink {
-		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-		if err != nil {
-			return err
-		}
-		// Pre-allocate the file for performance reasons
-		err = preAllocate(src.Size(), f)
-		if err != nil {
-			fs.Debugf(o, "Failed to pre-allocate: %v", err)
-		}
-		out = f
-	} else {
-		out = nopWriterCloser{&symlinkData}
+	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return err
+	}
+
+	// Pre-allocate the file for performance reasons
+	err = preAllocate(src.Size(), out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
 
 	// Calculate the hash of the object we are reading as we go along
@@ -955,26 +852,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if err == nil {
 		err = closeErr
 	}
-
-	if o.translatedLink {
-		if err == nil {
-			// Remove any current symlink or file, if one exists
-			if _, err := os.Lstat(o.path); err == nil {
-				if removeErr := os.Remove(o.path); removeErr != nil {
-					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
-					return removeErr
-				}
-			}
-			// Use the contents for the copied object to create a symlink
-			err = os.Symlink(symlinkData.String(), o.path)
-		}
-
-		// only continue if symlink creation succeeded
-		if err != nil {
-			return err
-		}
-	}
-
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
 		if removeErr := os.Remove(o.path); removeErr != nil {

backend/local/local_internal_test.go

@@ -1,16 +1,10 @@
 package local
 
 import (
-	"io/ioutil"
-	"os"
 	"path"
-	"path/filepath"
-	"runtime"
 	"testing"
 	"time"
 
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
 	"github.com/ncw/rclone/lib/file"
@@ -48,9 +42,6 @@ func TestUpdatingCheck(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}
-	defer func() {
-		require.NoError(t, fd.Close())
-	}()
 
 	fi, err := fd.Stat()
 	require.NoError(t, err)
@@ -81,108 +72,3 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 
 }
-
-func TestSymlink(t *testing.T) {
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	f := r.Flocal.(*Fs)
-	dir := f.root
-
-	// Write a file
-	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
-	file1 := r.WriteFile("file.txt", "hello", modTime1)
-
-	// Write a symlink
-	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
-	symlinkPath := filepath.Join(dir, "symlink.txt")
-	require.NoError(t, os.Symlink("file.txt", symlinkPath))
-	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
-
-	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
-	if runtime.GOOS == "windows" {
-		file2.Size = 0 // symlinks are 0 length under Windows
-	}
-
-	// Object viewed as destination
-	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
-
-	// Check with no symlink flags
-	fstest.CheckItems(t, r.Flocal, file1)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-L" mode
-	f.opt.FollowSymlinks = true
-	f.opt.TranslateSymlinks = false
-	f.lstat = os.Stat
-
-	fstest.CheckItems(t, r.Flocal, file1, file2d)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-l" mode
-	f.opt.FollowSymlinks = false
-	f.opt.TranslateSymlinks = true
-	f.lstat = os.Lstat
-
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2)
-	}
-
-	// Create a symlink
-	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
-	if runtime.GOOS == "windows" {
-		file3.Size = 0 // symlinks are 0 length under Windows
-	}
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
-	}
-
-	// Check it got the correct contents
-	symlinkPath = filepath.Join(dir, "symlink2.txt")
-	fi, err := os.Lstat(symlinkPath)
-	require.NoError(t, err)
-	assert.False(t, fi.Mode().IsRegular())
-	linkText, err := os.Readlink(symlinkPath)
-	require.NoError(t, err)
-	assert.Equal(t, "file.txt", linkText)
-
-	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
-	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
-	if runtime.GOOS != "windows" {
-		assert.Equal(t, int64(8), o.Size())
-	}
-
-	// Check that NewObject doesn't see the non suffixed version
-	_, err = r.Flocal.NewObject("symlink2.txt")
-	require.Equal(t, fs.ErrorObjectNotFound, err)
-
-	// Check reading the object
-	in, err := o.Open()
-	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt", string(contents))
-	require.NoError(t, in.Close())
-
-	// Check reading the object with range
-	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
-	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt"[2:5+1], string(contents))
-	require.NoError(t, in.Close())
-}
-
-func TestSymlinkError(t *testing.T) {
-	m := configmap.Simple{
-		"links":      "true",
-		"copy_links": "true",
-	}
-	_, err := NewFs("local", "/", m)
-	assert.Equal(t, errLinksAndCopyLinks, err)
-}

backend/local/read_device_unix.go

@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
 		fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
 		return devUnset
 	}
-	return uint64(statT.Dev) // nolint: unconvert
+	return uint64(statT.Dev)
 }

backend/mega/mega.go

@@ -98,7 +98,7 @@ type Fs struct {
 	opt        Options      // parsed config options
 	features   *fs.Features // optional features
 	srv        *mega.Mega   // the connection to the server
-	pacer      *fs.Pacer    // pacer for API calls
+	pacer      *pacer.Pacer // pacer for API calls
 	rootNodeMu sync.Mutex   // mutex for _rootNode
 	_rootNode  *mega.Node   // root node - call findRoot to use this
 	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir
@@ -217,7 +217,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   srv,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
@@ -497,7 +497,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, object, leaf and error
+// Returns the dirNode, obect, leaf and error
 //
 // Used to create new objects
 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -523,10 +523,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -847,14 +847,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.deleteNode(srcDirNode)
 		if err != nil {
-			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1076,9 +1076,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
-	if size < 0 {
-		return errors.New("mega backend can't upload a file of unknown length")
-	}
 	//modTime := src.ModTime()
 	remote := o.Remote()
 
@@ -1129,7 +1126,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return errors.Wrap(err, "failed to finish upload")
 	}
 
-	// If the upload succeeded and the original object existed, then delete it
+	// If the upload succeded and the original object existed, then delete it
 	if o.info != nil {
 		err = o.fs.deleteNode(o.info)
 		if err != nil {

backend/onedrive/api/types.go

@@ -25,7 +25,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := e.ErrorInfo.Code
 	if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +35,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Identity represents an identity of an actor. For example, and actor
@@ -295,9 +295,9 @@ func (i *Item) GetID() string {
 	return i.ID
 }
 
-// GetDriveID returns a normalized ParentReference of the item
+// GetDriveID returns a normalized ParentReferance of the item
 func (i *Item) GetDriveID() string {
-	return i.GetParentReference().DriveID
+	return i.GetParentReferance().DriveID
 }
 
 // GetName returns a normalized Name of the item
@@ -398,8 +398,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
 	return i.LastModifiedDateTime
 }
 
-// GetParentReference returns a normalized ParentReference of the item
-func (i *Item) GetParentReference() *ItemReference {
+// GetParentReferance returns a normalized ParentReferance of the item
+func (i *Item) GetParentReferance() *ItemReference {
 	if i.IsRemote() && i.ParentReference == nil {
 		return i.RemoteItem.ParentReference
 	}

backend/onedrive/onedrive.go

@@ -227,7 +227,7 @@ that the chunks will be buffered into memory.`,
 			Advanced: true,
 		}, {
 			Name:     "drive_type",
-			Help:     "The type of the drive ( " + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + " )",
+			Help:     "The type of the drive ( personal | business | documentLibrary )",
 			Default:  "",
 			Advanced: true,
 		}, {
@@ -261,7 +261,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the one drive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
@@ -324,24 +324,19 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
+	authRety := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
+		authRety = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
 // if `relPath` == "", it reads the metadata for the item with that ID.
-//
-// We address items using the pattern `drives/driveID/items/itemID:/relativePath`
-// instead of simply using `drives/driveID/root:/itemPath` because it works for
-// "shared with me" folders in OneDrive Personal (See #2536, #2778)
-// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
+	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -480,7 +475,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		driveID:   opt.DriveID,
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
-		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:     pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -497,11 +492,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// Get rootID
 	rootInfo, _, err := f.readMetaDataForPath("")
-	if err != nil || rootInfo.GetID() == "" {
+	if err != nil || rootInfo.ID == "" {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
 
-	f.dirCache = dircache.New(root, rootInfo.GetID(), f)
+	f.dirCache = dircache.New(root, rootInfo.ID, f)
 
 	// Find the current root
 	err = f.dirCache.FindRoot(false)
@@ -708,7 +703,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			id := info.GetID()
 			f.dirCache.Put(remote, id)
 			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-			d.SetItems(folder.ChildCount)
+			if folder != nil {
+				d.SetItems(folder.ChildCount)
+			}
 			entries = append(entries, d)
 		} else {
 			o, err := f.newObjectWithInfo(remote, info)
@@ -822,6 +819,9 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -1340,12 +1340,12 @@ func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
 		}
 	} else {
 		opts = rest.Opts{
 			Method: "PATCH",
-			Path:   "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
+			Path:   "/root:/" + rest.URLPathEscape(o.srvPath()),
 		}
 	}
 	update := api.SetFileSystemInfo{
@@ -1488,7 +1488,7 @@ func (o *Object) cancelUploadSession(url string) (err error) {
 // uploadMultipart uploads a file using multipart upload
 func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size <= 0 {
-		return nil, errors.New("unknown-sized upload not supported")
+		panic("size passed into uploadMultipart must be > 0")
 	}
 
 	// Create upload session
@@ -1535,7 +1535,7 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
 // This function will set modtime after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
-		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
+		panic("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -1560,6 +1560,10 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 		}
 	}
 
+	if size == 0 {
+		opts.Body = nil
+	}
+
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1602,7 +1606,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	} else if size == 0 {
 		info, err = o.uploadSinglepart(in, size, modTime)
 	} else {
-		return errors.New("unknown-sized upload not supported")
+		panic("src file size must be >= 0")
 	}
 	if err != nil {
 		return err
@@ -1668,21 +1672,6 @@ func getRelativePathInsideBase(base, target string) (string, bool) {
 	return "", false
 }
 
-// Adds a ":" at the end of `remotePath` in a proper manner.
-// If `remotePath` already ends with "/", change it to ":/"
-// If `remotePath` is "", return "".
-// A workaround for #2720 and #3039
-func withTrailingColon(remotePath string) string {
-	if remotePath == "" {
-		return ""
-	}
-
-	if strings.HasSuffix(remotePath, "/") {
-		return remotePath[:len(remotePath)-1] + ":/"
-	}
-	return remotePath + ":"
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -65,7 +65,7 @@ type Fs struct {
 	opt      Options            // parsed options
 	features *fs.Features       // optional features
 	srv      *rest.Client       // the connection to the server
-	pacer    *fs.Pacer          // To pace and retry the API calls
+	pacer    *pacer.Pacer       // To pace and retry the API calls
 	session  UserSessionInfo    // contains the session data
 	dirCache *dircache.DirCache // Map of directory path to directory id
 }
@@ -119,7 +119,7 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -144,7 +144,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 
 	f.dirCache = dircache.New(root, "0", f)
@@ -287,6 +287,9 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -782,7 +785,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, folder.FolderID)
-		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
+		d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
 		entries = append(entries, d)
 	}

backend/opendrive/types.go

@@ -13,7 +13,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }

backend/pcloud/api/types.go

@@ -41,7 +41,7 @@ type Error struct {
 	ErrorString string `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
 }
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
 	return e
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Item describes a folder or a file as returned by Get Folder Items and others
@@ -161,6 +161,7 @@ type UserInfo struct {
 	PublicLinkQuota       int64  `json:"publiclinkquota"`
 	Email                 string `json:"email"`
 	UserID                int    `json:"userid"`
+	Result                int    `json:"result"`
 	Quota                 int64  `json:"quota"`
 	TrashRevretentionDays int    `json:"trashrevretentiondays"`
 	Premium               bool   `json:"premium"`

backend/pcloud/pcloud.go

@@ -95,7 +95,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
@@ -254,7 +254,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         false,
@@ -385,7 +385,7 @@ func fileIDtoNumber(fileID string) string {
 	if len(fileID) > 0 && fileID[0] == 'f' {
 		return fileID[1:]
 	}
-	fs.Debugf(nil, "Invalid file id %q", fileID)
+	fs.Debugf(nil, "Invalid filee id %q", fileID)
 	return fileID
 }
 

backend/qingstor/qingstor.go

@@ -101,13 +101,10 @@ enough memory, then increasing this will speed up the transfers.`,
 This is the number of chunks of the same file that are uploaded
 concurrently.
 
-NB if you set this to > 1 then the checksums of multpart uploads
-become corrupted (the uploads themselves are not corrupted though).
-
 If you are uploading small numbers of large file over high speed link
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  1,
+			Default:  4,
 			Advanced: true,
 		}},
 	})
@@ -449,7 +446,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	_, err = bucketInit.PutObject(key, &req)
 	if err != nil {
-		fs.Debugf(f, "Copy Failed, API Error: %v", err)
+		fs.Debugf(f, "Copied Faild, API Error: %v", err)
 		return nil, err
 	}
 	return f.NewObject(remote)
@@ -756,7 +753,7 @@ func (f *Fs) Mkdir(dir string) error {
 		}
 		switch *statistics.Status {
 		case "deleted":
-			fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
+			fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
 			time.Sleep(time.Second * 1)
 			retries++
 			continue
@@ -875,7 +872,7 @@ func (o *Object) readMetaData() (err error) {
 	fs.Debugf(o, "Read metadata of key: %s", key)
 	resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
 	if err != nil {
-		fs.Debugf(o, "Read metadata failed, API Error: %v", err)
+		fs.Debugf(o, "Read metadata faild, API Error: %v", err)
 		if e, ok := err.(*qsErr.QingStorError); ok {
 			if e.StatusCode == http.StatusNotFound {
 				return fs.ErrorObjectNotFound

backend/qingstor/upload.go

@@ -143,7 +143,7 @@ func (u *uploader) init() {
 
 		// Try to adjust partSize if it is too small and account for
 		// integer division truncation.
-		if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
+		if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
 			// Add one to the part size to account for remainders
 			// during the size calculation. e.g odd number of bytes.
 			u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -163,7 +163,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 
 	_, err := bucketInit.PutObject(u.cfg.key, &req)
 	if err == nil {
-		fs.Debugf(u, "Upload single object finished")
+		fs.Debugf(u, "Upload single objcet finished")
 	}
 	return err
 }

backend/s3/s3.go

@@ -131,9 +131,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -237,10 +234,10 @@ func init() {
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.softlayer.net",
-				Help:  "Great Britain Endpoint",
+				Help:  "Great Britan Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
-				Help:  "Great Britain Private Endpoint",
+				Help:  "Great Britan Private Endpoint",
 			}, {
 				Value: "s3.ap-geo.objectstorage.softlayer.net",
 				Help:  "APAC Cross Regional Endpoint",
@@ -346,7 +343,7 @@ func init() {
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
 			Provider: "!AWS,IBMCOS,Alibaba",
 			Examples: []fs.OptionExample{{
-				Value:    "objects-us-east-1.dream.io",
+				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
 			}, {
@@ -395,9 +392,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -450,7 +444,7 @@ func init() {
 				Help:  "US East Region Flex",
 			}, {
 				Value: "us-south-standard",
-				Help:  "US South Region Standard",
+				Help:  "US Sout hRegion Standard",
 			}, {
 				Value: "us-south-vault",
 				Help:  "US South Region Vault",
@@ -474,16 +468,16 @@ func init() {
 				Help:  "EU Cross Region Flex",
 			}, {
 				Value: "eu-gb-standard",
-				Help:  "Great Britain Standard",
+				Help:  "Great Britan Standard",
 			}, {
 				Value: "eu-gb-vault",
-				Help:  "Great Britain Vault",
+				Help:  "Great Britan Vault",
 			}, {
 				Value: "eu-gb-cold",
-				Help:  "Great Britain Cold",
+				Help:  "Great Britan Cold",
 			}, {
 				Value: "eu-gb-flex",
-				Help:  "Great Britain Flex",
+				Help:  "Great Britan Flex",
 			}, {
 				Value: "ap-standard",
 				Help:  "APAC Standard",
@@ -782,7 +776,7 @@ type Fs struct {
 	bucketOKMu    sync.Mutex       // mutex to protect bucket OK
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
-	pacer         *fs.Pacer        // To pace the API calls
+	pacer         *pacer.Pacer     // To pace the API calls
 	srv           *http.Client     // a plain http client
 }
 
@@ -839,24 +833,15 @@ var retryErrorCodes = []int{
 //S3 is pretty resilient, and the built in retry handling is probably sufficient
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func shouldRetry(err error) (bool, error) {
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
-		// Simple case, check the original embedded error in case it's generically retryable
+		// Simple case, check the original embedded error in case it's generically retriable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
 		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
-			// 301 if wrong region for bucket
-			if reqErr.StatusCode() == http.StatusMovedPermanently {
-				urfbErr := f.updateRegionForBucket()
-				if urfbErr != nil {
-					fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
-					return false, err
-				}
-				return true, err
-			}
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
 					return true, err
@@ -945,17 +930,12 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
+		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
+		WithEndpoint(opt.Endpoint).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle)
-	if opt.Region != "" {
-		awsConfig.WithRegion(opt.Region)
-	}
-	if opt.Endpoint != "" {
-		awsConfig.WithEndpoint(opt.Endpoint)
-	}
-
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
 	awsSessionOpts := session.Options{
 		Config: *awsConfig,
@@ -1055,7 +1035,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		c:      c,
 		bucket: bucket,
 		ses:    ses,
-		pacer:  fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
 		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1072,7 +1052,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.c.HeadObject(&req)
-			return f.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err == nil {
 			f.root = path.Dir(directory)
@@ -1122,51 +1102,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// Gets the bucket location
-func (f *Fs) getBucketLocation() (string, error) {
-	req := s3.GetBucketLocationInput{
-		Bucket: &f.bucket,
-	}
-	var resp *s3.GetBucketLocationOutput
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(err)
-	})
-	if err != nil {
-		return "", err
-	}
-	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
-}
-
-// Updates the region for the bucket by reading the region from the
-// bucket then updating the session.
-func (f *Fs) updateRegionForBucket() error {
-	region, err := f.getBucketLocation()
-	if err != nil {
-		return errors.Wrap(err, "reading bucket location failed")
-	}
-	if aws.StringValue(f.c.Config.Endpoint) != "" {
-		return errors.Errorf("can't set region to %q as endpoint is set", region)
-	}
-	if aws.StringValue(f.c.Config.Region) == region {
-		return errors.Errorf("region is already %q - not updating", region)
-	}
-
-	// Make a new session with the new region
-	oldRegion := f.opt.Region
-	f.opt.Region = region
-	c, ses, err := s3Connection(&f.opt)
-	if err != nil {
-		return errors.Wrap(err, "creating new session failed")
-	}
-	f.c = c
-	f.ses = ses
-
-	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
-	return nil
-}
-
 // listFn is called from list to handle an object.
 type listFn func(remote string, object *s3.Object, isDirectory bool) error
 
@@ -1199,7 +1134,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListObjects(&req)
-			return f.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1328,7 +1263,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	var resp *s3.ListBucketsOutput
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.ListBuckets(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1416,7 +1351,7 @@ func (f *Fs) dirExists() (bool, error) {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.HeadBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err == nil {
 		return true, nil
@@ -1456,7 +1391,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CreateBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err, ok := err.(awserr.Error); ok {
 		if err.Code() == "BucketAlreadyOwnedByYou" {
@@ -1485,7 +1420,7 @@ func (f *Fs) Rmdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.DeleteBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err == nil {
 		f.bucketOK = false
@@ -1546,7 +1481,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.c.CopyObject(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1628,7 +1563,7 @@ func (o *Object) readMetaData() (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1724,7 +1659,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.CopyObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	return err
 }
@@ -1756,7 +1691,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.GetObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
@@ -1847,7 +1782,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		}
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			_, err = uploader.Upload(&req)
-			return o.fs.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return err
@@ -1903,11 +1838,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			resp, err := o.fs.srv.Do(httpReq)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return shouldRetry(err)
 			}
 			body, err := rest.ReadBody(resp)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return shouldRetry(err)
 			}
 			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
 				return false, nil
@@ -1935,7 +1870,7 @@ func (o *Object) Remove() error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.DeleteObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	return err
 }

backend/sftp/sftp.go

@@ -427,12 +427,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
-	return NewFsWithConnection(name, root, opt, sshConfig)
-}
-
-// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
-// the host specified in the ssh.ClientConfig
-func NewFsWithConnection(name string, root string, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
 		name:      name,
 		root:      root,
@@ -571,13 +565,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
 		if !info.Mode().IsRegular() {
-			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
-				if !os.IsNotExist(err) {
-					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
-				}
-				info = oldInfo
+				return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
 			}
 		}
 		if info.IsDir() {

backend/swift/auth.go

@@ -2,7 +2,6 @@ package swift
 
 import (
 	"net/http"
-	"time"
 
 	"github.com/ncw/swift"
 )
@@ -66,14 +65,6 @@ func (a *auth) Token() string {
 	return a.parentAuth.Token()
 }
 
-// Expires returns the time the token expires if known or Zero if not.
-func (a *auth) Expires() (t time.Time) {
-	if do, ok := a.parentAuth.(swift.Expireser); ok {
-		t = do.Expires()
-	}
-	return t
-}
-
 // The CDN url if available
 func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
@@ -83,7 +74,4 @@ func (a *auth) CdnUrl() string { // nolint
 }
 
 // Check the interfaces are satisfied
-var (
-	_ swift.Authenticator = (*auth)(nil)
-	_ swift.Expireser     = (*auth)(nil)
-)
+var _ swift.Authenticator = (*auth)(nil)

backend/swift/swift.go

@@ -130,15 +130,6 @@ func init() {
 		}, {
 			Name: "auth_token",
 			Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
-		}, {
-			Name: "application_credential_id",
-			Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)",
-		}, {
-			Name: "application_credential_name",
-			Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)",
-		}, {
-			Name: "application_credential_secret",
-			Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)",
 		}, {
 			Name:    "auth_version",
 			Help:    "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
@@ -182,26 +173,23 @@ provider.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth                     bool          `config:"env_auth"`
-	User                        string        `config:"user"`
-	Key                         string        `config:"key"`
-	Auth                        string        `config:"auth"`
-	UserID                      string        `config:"user_id"`
-	Domain                      string        `config:"domain"`
-	Tenant                      string        `config:"tenant"`
-	TenantID                    string        `config:"tenant_id"`
-	TenantDomain                string        `config:"tenant_domain"`
-	Region                      string        `config:"region"`
-	StorageURL                  string        `config:"storage_url"`
-	AuthToken                   string        `config:"auth_token"`
-	AuthVersion                 int           `config:"auth_version"`
-	ApplicationCredentialID     string        `config:"application_credential_id"`
-	ApplicationCredentialName   string        `config:"application_credential_name"`
-	ApplicationCredentialSecret string        `config:"application_credential_secret"`
-	StoragePolicy               string        `config:"storage_policy"`
-	EndpointType                string        `config:"endpoint_type"`
-	ChunkSize                   fs.SizeSuffix `config:"chunk_size"`
-	NoChunk                     bool          `config:"no_chunk"`
+	EnvAuth       bool          `config:"env_auth"`
+	User          string        `config:"user"`
+	Key           string        `config:"key"`
+	Auth          string        `config:"auth"`
+	UserID        string        `config:"user_id"`
+	Domain        string        `config:"domain"`
+	Tenant        string        `config:"tenant"`
+	TenantID      string        `config:"tenant_id"`
+	TenantDomain  string        `config:"tenant_domain"`
+	Region        string        `config:"region"`
+	StorageURL    string        `config:"storage_url"`
+	AuthToken     string        `config:"auth_token"`
+	AuthVersion   int           `config:"auth_version"`
+	StoragePolicy string        `config:"storage_policy"`
+	EndpointType  string        `config:"endpoint_type"`
+	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
+	NoChunk       bool          `config:"no_chunk"`
 }
 
 // Fs represents a remote swift server
@@ -216,7 +204,7 @@ type Fs struct {
 	containerOK       bool              // true if we have created the container
 	segmentsContainer string            // container to store the segments (if any) in
 	noCheckContainer  bool              // don't check the container before creating it
-	pacer             *fs.Pacer         // To pace the API calls
+	pacer             *pacer.Pacer      // To pace the API calls
 }
 
 // Object describes a swift object
@@ -305,25 +293,22 @@ func parsePath(path string) (container, directory string, err error) {
 func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	c := &swift.Connection{
 		// Keep these in the same order as the Config for ease of checking
-		UserName:                    opt.User,
-		ApiKey:                      opt.Key,
-		AuthUrl:                     opt.Auth,
-		UserId:                      opt.UserID,
-		Domain:                      opt.Domain,
-		Tenant:                      opt.Tenant,
-		TenantId:                    opt.TenantID,
-		TenantDomain:                opt.TenantDomain,
-		Region:                      opt.Region,
-		StorageUrl:                  opt.StorageURL,
-		AuthToken:                   opt.AuthToken,
-		AuthVersion:                 opt.AuthVersion,
-		ApplicationCredentialId:     opt.ApplicationCredentialID,
-		ApplicationCredentialName:   opt.ApplicationCredentialName,
-		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
-		EndpointType:                swift.EndpointType(opt.EndpointType),
-		ConnectTimeout:              10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:                     10 * fs.Config.Timeout,        // Use the timeouts in the transport
-		Transport:                   fshttp.NewTransport(fs.Config),
+		UserName:       opt.User,
+		ApiKey:         opt.Key,
+		AuthUrl:        opt.Auth,
+		UserId:         opt.UserID,
+		Domain:         opt.Domain,
+		Tenant:         opt.Tenant,
+		TenantId:       opt.TenantID,
+		TenantDomain:   opt.TenantDomain,
+		Region:         opt.Region,
+		StorageUrl:     opt.StorageURL,
+		AuthToken:      opt.AuthToken,
+		AuthVersion:    opt.AuthVersion,
+		EndpointType:   swift.EndpointType(opt.EndpointType),
+		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:      fshttp.NewTransport(fs.Config),
 	}
 	if opt.EnvAuth {
 		err := c.ApplyEnvironment()
@@ -333,13 +318,11 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	}
 	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
-		if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
-			if c.UserName == "" && c.UserId == "" {
-				return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
-			}
-			if c.ApiKey == "" {
-				return nil, errors.New("key not found")
-			}
+		if c.UserName == "" && c.UserId == "" {
+			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
+		}
+		if c.ApiKey == "" {
+			return nil, errors.New("key not found")
 		}
 		if c.AuthUrl == "" {
 			return nil, errors.New("auth not found")
@@ -401,7 +384,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		segmentsContainer: container + "_segments",
 		root:              directory,
 		noCheckContainer:  noCheckContainer,
-		pacer:             fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+		pacer:             pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -430,7 +413,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 	return f, nil
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)

backend/union/union.go

@@ -177,8 +177,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // At least one value will be written to the channel,
 // specifying the initial value and updated values might
 // follow. A 0 Duration should pause the polling.
-// The ChangeNotify implementation must empty the channel
-// regularly. When the channel gets closed, the implementation
+// The ChangeNotify implemantion must empty the channel
+// regulary. When the channel gets closed, the implemantion
 // should stop polling and release resources.
 func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
 	var remoteChans []chan time.Duration
@@ -376,11 +376,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	features = features.Mask(f.wr) // mask the features just on the writable fs
 
-	// Really need the union of all remotes for these, so
-	// re-instate and calculate separately.
-	features.ChangeNotify = f.ChangeNotify
-	features.DirCacheFlush = f.DirCacheFlush
-
 	// FIXME maybe should be masking the bools here?
 
 	// Clear ChangeNotify and DirCacheFlush if all are nil

backend/webdav/api/types.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/hash"
 )
 
 const (
@@ -66,13 +65,11 @@ type Response struct {
 // Note that status collects all the status values for which we just
 // check the first is OK.
 type Prop struct {
-	Status       []string  `xml:"DAV: status"`
-	Name         string    `xml:"DAV: prop>displayname,omitempty"`
-	Type         *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
-	IsCollection *string   `xml:"DAV: prop>iscollection,omitempty"` // this is a Microsoft extension see #2716
-	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
-	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
-	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
+	Status   []string  `xml:"DAV: status"`
+	Name     string    `xml:"DAV: prop>displayname,omitempty"`
+	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -98,33 +95,13 @@ func (p *Prop) StatusOK() bool {
 	return false
 }
 
-// Hashes returns a map of all checksums - may be nil
-func (p *Prop) Hashes() (hashes map[hash.Type]string) {
-	if len(p.Checksums) == 0 {
-		return nil
-	}
-	hashes = make(map[hash.Type]string)
-	for _, checksums := range p.Checksums {
-		checksums = strings.ToLower(checksums)
-		for _, checksum := range strings.Split(checksums, " ") {
-			switch {
-			case strings.HasPrefix(checksum, "sha1:"):
-				hashes[hash.SHA1] = checksum[5:]
-			case strings.HasPrefix(checksum, "md5:"):
-				hashes[hash.MD5] = checksum[4:]
-			}
-		}
-	}
-	return hashes
-}
-
 // PropValue is a tagged name and value
 type PropValue struct {
 	XMLName xml.Name `xml:""`
 	Value   string   `xml:",chardata"`
 }
 
-// Error is used to describe webdav errors
+// Error is used to desribe webdav errors
 //
 // <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
 //   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
@@ -137,7 +114,7 @@ type Error struct {
 	StatusCode int
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	var out []string
 	if e.Message != "" {
@@ -210,22 +187,3 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	}
 	return err
 }
-
-// Quota is used to read the bytes used and available
-//
-// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
-//  <d:response>
-//   <d:href>/remote.php/webdav/</d:href>
-//   <d:propstat>
-//    <d:prop>
-//     <d:quota-available-bytes>-3</d:quota-available-bytes>
-//     <d:quota-used-bytes>376461895</d:quota-used-bytes>
-//    </d:prop>
-//    <d:status>HTTP/1.1 200 OK</d:status>
-//   </d:propstat>
-//  </d:response>
-// </d:multistatus>
-type Quota struct {
-	Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
-	Used      int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
-}

backend/webdav/odrvcookie/fetch.go

@@ -102,7 +102,7 @@ func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
 func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
 	spRoot, err := url.Parse(ca.endpoint)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
+		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
 	}
 
 	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
@@ -121,7 +121,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 		Jar: jar,
 	}
 
-	// Send the previously acquired Token as a Post parameter
+	// Send the previously aquired Token as a Post parameter
 	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
 		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
 	}

backend/webdav/odrvcookie/renew.go

@@ -2,10 +2,13 @@ package odrvcookie
 
 import (
 	"time"
+
+	"github.com/ncw/rclone/lib/rest"
 )
 
 // CookieRenew holds information for the renew
 type CookieRenew struct {
+	srv     *rest.Client
 	timer   *time.Ticker
 	renewFn func()
 }

backend/webdav/webdav.go

@@ -2,13 +2,23 @@
 // object storage system.
 package webdav
 
+// Owncloud: Getting Oc-Checksum:
+// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
+// nextcloud?
+
+// docs for file webdav
+// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
+
+// indicates checksums can be set as metadata here
+// https://github.com/nextcloud/server/issues/6129
+// owncloud seems to have checksums as metadata though - can read them
+
 // SetModTime might be possible
 // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
 // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
 // For example the ownCloud WebDAV server does it that way.
 
 import (
-	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -101,12 +111,11 @@ type Fs struct {
 	endpoint           *url.URL      // URL of the host
 	endpointURL        string        // endpoint as a string
 	srv                *rest.Client  // the connection to the one drive server
-	pacer              *fs.Pacer     // pacer for API calls
+	pacer              *pacer.Pacer  // pacer for API calls
 	precision          time.Duration // mod time precision
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
-	hasChecksums       bool          // set if can use owncloud style checksums
 }
 
 // Object describes a webdav object
@@ -118,8 +127,7 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	sha1        string    // SHA-1 of the object content if known
-	md5         string    // MD5 of the object content if known
+	sha1        string    // SHA-1 of the object content
 }
 
 // ------------------------------------------------------------
@@ -172,18 +180,6 @@ func itemIsDir(item *api.Response) bool {
 		}
 		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
 	}
-	// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
-	// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
-	if t := item.Props.IsCollection; t != nil {
-		switch x := strings.ToLower(*t); x {
-		case "0", "false":
-			return false
-		case "1", "true":
-			return true
-		default:
-			fs.Debugf(nil, "Unknown value %q for IsCollection", x)
-		}
-	}
 	return false
 }
 
@@ -198,9 +194,6 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
 		},
 		NoRedirect: true,
 	}
-	if f.hasChecksums {
-		opts.Body = bytes.NewBuffer(owncloudProps)
-	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -256,7 +249,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// addSlash makes sure s is terminated with a / if non empty
+// addShlash makes sure s is terminated with a / if non empty
 func addSlash(s string) string {
 	if s != "" && !strings.HasSuffix(s, "/") {
 		s += "/"
@@ -318,7 +311,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		endpoint:    u,
 		endpointURL: u.String(),
 		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
-		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		precision:   fs.ModTimeNotSupported,
 	}
 	f.features = (&fs.Features{
@@ -364,11 +357,9 @@ func (f *Fs) setQuirks(vendor string) error {
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -435,22 +426,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// Read the normal props, plus the checksums
-//
-// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
-var owncloudProps = []byte(`<?xml version="1.0"?>
-<d:propfind  xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
- <d:prop>
-  <d:displayname />
-  <d:getlastmodified />
-  <d:getcontentlength />
-  <d:resourcetype />
-  <d:getcontenttype />
-  <oc:checksums />
- </d:prop>
-</d:propfind>
-`)
-
 // list the objects into the function supplied
 //
 // If directories is set it only sends directories
@@ -470,9 +445,6 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
 			"Depth": depth,
 		},
 	}
-	if f.hasChecksums {
-		opts.Body = bytes.NewBuffer(owncloudProps)
-	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -644,18 +616,10 @@ func (f *Fs) _mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	err := f.pacer.Call(func() (bool, error) {
+	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
-	if apiErr, ok := err.(*api.Error); ok {
-		// already exists
-		// owncloud returns 423/StatusLocked if the create is already in progress
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
-			return nil
-		}
-	}
-	return err
 }
 
 // mkdir makes the directory and parents using native paths
@@ -663,7 +627,11 @@ func (f *Fs) mkdir(dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
-		// parent does not exist so create it first then try again
+		// already exists
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
+			return nil
+		}
+		// parent does not exist
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
@@ -879,54 +847,9 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	if f.hasChecksums {
-		return hash.NewHashSet(hash.MD5, hash.SHA1)
-	}
 	return hash.Set(hash.None)
 }
 
-// About gets quota information
-func (f *Fs) About() (*fs.Usage, error) {
-	opts := rest.Opts{
-		Method: "PROPFIND",
-		Path:   "",
-		ExtraHeaders: map[string]string{
-			"Depth": "0",
-		},
-	}
-	opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
-<D:propfind xmlns:D="DAV:">
- <D:prop>
-  <D:quota-available-bytes/>
-  <D:quota-used-bytes/>
- </D:prop>
-</D:propfind>
-`))
-	var q = api.Quota{
-		Available: -1,
-		Used:      -1,
-	}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &q)
-		return shouldRetry(resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "about call failed")
-	}
-	usage := &fs.Usage{}
-	if q.Available != 0 || q.Used != 0 {
-		if q.Available >= 0 && q.Used >= 0 {
-			usage.Total = fs.NewUsageValue(q.Available + q.Used)
-		}
-		if q.Used >= 0 {
-			usage.Used = fs.NewUsageValue(q.Used)
-		}
-	}
-	return usage, nil
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -947,17 +870,12 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
+// Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(t hash.Type) (string, error) {
-	if o.fs.hasChecksums {
-		switch t {
-		case hash.SHA1:
-			return o.sha1, nil
-		case hash.MD5:
-			return o.md5, nil
-		}
+	if t != hash.SHA1 {
+		return "", hash.ErrUnsupported
 	}
-	return "", hash.ErrUnsupported
+	return o.sha1, nil
 }
 
 // Size returns the size of an object in bytes
@@ -975,11 +893,6 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
-	if o.fs.hasChecksums {
-		hashes := info.Hashes()
-		o.sha1 = hashes[hash.SHA1]
-		o.md5 = hashes[hash.MD5]
-	}
 	return nil
 }
 
@@ -1059,21 +972,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
 		ContentType:   fs.MimeType(src),
 	}
-	if o.fs.useOCMtime || o.fs.hasChecksums {
-		opts.ExtraHeaders = map[string]string{}
-		if o.fs.useOCMtime {
-			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
-		}
-		if o.fs.hasChecksums {
-			// Set an upload checksum - prefer SHA1
-			//
-			// This is used as an upload integrity test. If we set
-			// only SHA1 here, owncloud will calculate the MD5 too.
-			if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
-			} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
-			}
+	if o.fs.useOCMtime {
+		opts.ExtraHeaders = map[string]string{
+			"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
 		}
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1117,6 +1018,5 @@ var (
 	_ fs.Copier      = (*Fs)(nil)
 	_ fs.Mover       = (*Fs)(nil)
 	_ fs.DirMover    = (*Fs)(nil)
-	_ fs.Abouter     = (*Fs)(nil)
 	_ fs.Object      = (*Object)(nil)
 )

backend/yandex/api/types.go

@@ -56,7 +56,7 @@ type AsyncInfo struct {
 	Templated bool   `json:"templated"`
 }
 
-// AsyncStatus is returned when requesting the status of an async operations. Possible values in-progress, success, failure
+// AsyncStatus is returned when requesting the status of an async operations. Possble values in-progress, success, failure
 type AsyncStatus struct {
 	Status string `json:"status"`
 }

backend/yandex/yandex.go

@@ -93,7 +93,7 @@ type Fs struct {
 	opt      Options      // parsed options
 	features *fs.Features // optional features
 	srv      *rest.Client // the connection to the yandex server
-	pacer    *fs.Pacer    // pacer for API calls
+	pacer    *pacer.Pacer // pacer for API calls
 	diskRoot string       // root path with "disk:/" container name
 }
 
@@ -269,7 +269,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -307,7 +307,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs
 		if err != nil {
 			return nil, errors.Wrap(err, "error parsing time in directory item")
 		}
-		d := fs.NewDir(remote, t).SetSize(object.Size)
+		d := fs.NewDir(remote, t).SetSize(int64(object.Size))
 		return d, nil
 	case "file":
 		o, err := f.newObjectWithInfo(remote, object)
@@ -634,7 +634,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copies or moves directories or files depending on the method parameter
+// copyOrMoves copys or moves directories or files depending on the mthod parameter
 func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -1107,7 +1107,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	//if file uploaded successfully then return metadata
+	//if file uploaded sucessfully then return metadata
 	o.modTime = modTime
 	o.md5sum = ""                   // according to unit tests after put the md5 is empty.
 	o.size = int64(in1.BytesRead()) // better solution o.readMetaData() ?

bin/get-github-release.go

@@ -8,8 +8,6 @@
 package main
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -17,18 +15,13 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
-	"runtime"
 	"strings"
 	"time"
 
-	"github.com/ncw/rclone/lib/rest"
-	"golang.org/x/net/html"
 	"golang.org/x/sys/unix"
 )
 
@@ -37,15 +30,8 @@ var (
 	install = flag.Bool("install", false, "Install the downloaded package using sudo dpkg -i.")
 	extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.")
 	bindir  = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.")
-	useAPI  = flag.Bool("use-api", false, "Use the API for finding the release instead of scraping the page.")
 	// Globals
-	matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
-	osAliases    = map[string][]string{
-		"darwin": []string{"macos", "osx"},
-	}
-	archAliases = map[string][]string{
-		"amd64": []string{"x86_64"},
-	}
+	matchProject = regexp.MustCompile(`^(\w+)/(\w+)$`)
 )
 
 // A github release
@@ -127,41 +113,25 @@ func writable(path string) bool {
 
 // Directory to install releases in by default
 //
-// Find writable directories on $PATH.  Use $GOPATH/bin if that is on
-// the path and writable or use the first writable directory which is
-// in $HOME or failing that the first writable directory.
+// Find writable directories on $PATH.  Use the first writable
+// directory which is in $HOME or failing that the first writable
+// directory.
 //
 // Returns "" if none of the above were found
 func defaultBinDir() string {
 	home := os.Getenv("HOME")
-	var (
-		bin       string
-		homeBin   string
-		goHomeBin string
-		gopath    = os.Getenv("GOPATH")
-	)
+	var binDir string
 	for _, dir := range strings.Split(os.Getenv("PATH"), ":") {
 		if writable(dir) {
 			if strings.HasPrefix(dir, home) {
-				if homeBin != "" {
-					homeBin = dir
-				}
-				if gopath != "" && strings.HasPrefix(dir, gopath) && goHomeBin == "" {
-					goHomeBin = dir
-				}
+				return dir
 			}
-			if bin == "" {
-				bin = dir
+			if binDir != "" {
+				binDir = dir
 			}
 		}
 	}
-	if goHomeBin != "" {
-		return goHomeBin
-	}
-	if homeBin != "" {
-		return homeBin
-	}
-	return bin
+	return binDir
 }
 
 // read the body or an error message
@@ -205,8 +175,7 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	}
 
 	for _, asset := range release.Assets {
-		//log.Printf("Finding %s", asset.Name)
-		if matchName.MatchString(asset.Name) && isOurOsArch(asset.Name) {
+		if matchName.MatchString(asset.Name) {
 			return asset.BrowserDownloadURL, asset.Name
 		}
 	}
@@ -214,73 +183,6 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	return "", ""
 }
 
-// Get an asset URL and name by scraping the downloads page
-//
-// This doesn't use the API so isn't rate limited when not using GITHUB login details
-func getAssetFromReleasesPage(project string, matchName *regexp.Regexp) (assetURL string, assetName string) {
-	baseURL := "https://github.com/" + project + "/releases"
-	log.Printf("Fetching asset info for %q from %q", project, baseURL)
-	base, err := url.Parse(baseURL)
-	if err != nil {
-		log.Fatalf("URL Parse failed: %v", err)
-	}
-	resp, err := http.Get(baseURL)
-	if err != nil {
-		log.Fatalf("Failed to fetch release info %q: %v", baseURL, err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		log.Printf("Error: %s", readBody(resp.Body))
-		log.Fatalf("Bad status %d when fetching %q release info: %s", resp.StatusCode, baseURL, resp.Status)
-	}
-	doc, err := html.Parse(resp.Body)
-	if err != nil {
-		log.Fatalf("Failed to parse web page: %v", err)
-	}
-	var walk func(*html.Node)
-	walk = func(n *html.Node) {
-		if n.Type == html.ElementNode && n.Data == "a" {
-			for _, a := range n.Attr {
-				if a.Key == "href" {
-					if name := path.Base(a.Val); matchName.MatchString(name) && isOurOsArch(name) {
-						if u, err := rest.URLJoin(base, a.Val); err == nil {
-							if assetName == "" {
-								assetName = name
-								assetURL = u.String()
-							}
-						}
-					}
-					break
-				}
-			}
-		}
-		for c := n.FirstChild; c != nil; c = c.NextSibling {
-			walk(c)
-		}
-	}
-	walk(doc)
-	if assetName == "" || assetURL == "" {
-		log.Fatalf("Didn't find URL in page")
-	}
-	return assetURL, assetName
-}
-
-// isOurOsArch returns true if s contains our OS and our Arch
-func isOurOsArch(s string) bool {
-	s = strings.ToLower(s)
-	check := func(base string, aliases map[string][]string) bool {
-		names := []string{base}
-		names = append(names, aliases[base]...)
-		for _, name := range names {
-			if strings.Contains(s, name) {
-				return true
-			}
-		}
-		return false
-	}
-	return check(runtime.GOARCH, archAliases) && check(runtime.GOOS, osAliases)
-}
-
 // get a file for download
 func getFile(url, fileName string) {
 	log.Printf("Downloading %q from %q", fileName, url)
@@ -327,66 +229,6 @@ func run(args ...string) {
 	}
 }
 
-// Untars fileName from srcFile
-func untar(srcFile, fileName, extractDir string) {
-	f, err := os.Open(srcFile)
-	if err != nil {
-		log.Fatalf("Couldn't open tar: %v", err)
-	}
-	defer func() {
-		err := f.Close()
-		if err != nil {
-			log.Fatalf("Couldn't close tar: %v", err)
-		}
-	}()
-
-	var in io.Reader = f
-
-	srcExt := filepath.Ext(srcFile)
-	if srcExt == ".gz" || srcExt == ".tgz" {
-		gzf, err := gzip.NewReader(f)
-		if err != nil {
-			log.Fatalf("Couldn't open gzip: %v", err)
-		}
-		in = gzf
-	}
-
-	tarReader := tar.NewReader(in)
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			log.Fatalf("Trouble reading tar file: %v", err)
-		}
-		name := header.Name
-		switch header.Typeflag {
-		case tar.TypeReg:
-			baseName := filepath.Base(name)
-			if baseName == fileName {
-				outPath := filepath.Join(extractDir, fileName)
-				out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
-				if err != nil {
-					log.Fatalf("Couldn't open output file: %v", err)
-				}
-				defer func() {
-					err := out.Close()
-					if err != nil {
-						log.Fatalf("Couldn't close output: %v", err)
-					}
-				}()
-				n, err := io.Copy(out, tarReader)
-				if err != nil {
-					log.Fatalf("Couldn't write output file: %v", err)
-				}
-				log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
-			}
-		}
-	}
-}
-
 func main() {
 	flag.Parse()
 	args := flag.Args()
@@ -402,12 +244,7 @@ func main() {
 		log.Fatalf("Invalid regexp for name %q: %v", nameRe, err)
 	}
 
-	var assetURL, assetName string
-	if *useAPI {
-		assetURL, assetName = getAsset(project, matchName)
-	} else {
-		assetURL, assetName = getAssetFromReleasesPage(project, matchName)
-	}
+	assetURL, assetName := getAsset(project, matchName)
 	fileName := filepath.Join(os.TempDir(), assetName)
 	getFile(assetURL, fileName)
 
@@ -420,6 +257,8 @@ func main() {
 			log.Fatalf("Need to set -bindir")
 		}
 		log.Printf("Unpacking %s from %s and installing into %s", *extract, fileName, *bindir)
-		untar(fileName, *extract, *bindir+"/")
+		run("tar", "xf", fileName, *extract)
+		run("chmod", "a+x", *extract)
+		run("mv", "-f", *extract, *bindir+"/")
 	}
 }

bin/make_manual.py

@@ -36,7 +36,6 @@ docs = [
     "http.md",
     "hubic.md",
     "jottacloud.md",
-    "koofr.md",
     "mega.md",
     "azureblob.md",
     "onedrive.md",

bin/tidy-beta

@@ -3,7 +3,7 @@
 
 version="$1"
 if [ "$version" = "" ]; then
-    echo "Syntax: $0 <version, eg v1.42> [delete]"
+    echo "Syntax: $0 <version> [delete]"
     exit 1
 fi
 dry_run="--dry-run"
@@ -14,4 +14,4 @@ else
     echo "Use '$0 $version delete' to actually delete files"
 fi
 
-rclone ${dry_run} -P --fast-list --checkers 16 --transfers 16 delete --include "**${version}**" memstore:beta-rclone-org
+rclone ${dry_run} --fast-list -P --checkers 16 --transfers 16 delete --include "**/${version}**" memstore:beta-rclone-org

bin/upload-github

@@ -29,7 +29,7 @@ github-release release \
     --name "rclone" \
     --description "Rclone - rsync for cloud storage. Sync files to and from many cloud storage providers."
 
-for build in `ls build | grep -v current | grep -v testbuilds`; do
+for build in `ls build | grep -v current`; do
     echo "Uploading ${build}"
     base="${build%.*}"
     parts=(${base//-/ })

cmd/cmd.go

@@ -341,7 +341,8 @@ func initConfig() {
 	configflags.SetFlags()
 
 	// Load filters
-	err := filterflags.Reload()
+	var err error
+	filter.Active, err = filter.NewFilter(&filterflags.Opt)
 	if err != nil {
 		log.Fatalf("Failed to load filters: %v", err)
 	}
@@ -455,7 +456,7 @@ func AddBackendFlags() {
 					help = help[:nl]
 				}
 				help = strings.TrimSpace(help)
-				flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
+				flag := pflag.CommandLine.VarPF(opt, name, string(opt.ShortOpt), help)
 				if _, isBool := opt.Default.(bool); isBool {
 					flag.NoOptDefVal = "true"
 				}

cmd/copy/copy.go

@@ -7,13 +7,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var (
-	createEmptySrcDirs = false
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
-	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy")
 }
 
 var commandDefintion = &cobra.Command{
@@ -74,7 +69,7 @@ changed recently very efficiently like this:
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc, createEmptySrcDirs)
+				return sync.CopyDir(fdst, fsrc)
 			}
 			return operations.CopyFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/copyto/copyto.go

@@ -48,7 +48,7 @@ destination.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc, false)
+				return sync.CopyDir(fdst, fsrc)
 			}
 			return operations.CopyFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/help.go

@@ -32,48 +32,8 @@ documentation, changelog and configuration walkthroughs.
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
 	},
-	BashCompletionFunction: bashCompletionFunc,
 }
 
-const (
-	bashCompletionFunc = `
-__rclone_custom_func() {
-    if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
-        local cur cword prev words
-        if declare -F _init_completion > /dev/null; then
-            _init_completion -n : || return
-        else
-            __rclone_init_completion -n : || return
-        fi
-        if [[ $cur != *:* ]]; then
-            local remote
-            while IFS= read -r remote; do
-                [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
-            done < <(command rclone listremotes)
-            if [[ ${COMPREPLY[@]} ]]; then
-                local paths=("$cur"*)
-                [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
-            fi
-        else
-            local path=${cur#*:}
-            if [[ $path == */* ]]; then
-                local prefix=$(eval printf '%s' "${path%/*}")
-            else
-                local prefix=
-            fi
-            local line
-            while IFS= read -r line; do
-                local reply=${prefix:+$prefix/}$line
-                [[ $reply != $path* ]] || COMPREPLY+=("$reply")
-            done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
-	    [[ ! ${COMPREPLY[@]} ]] || compopt -o filenames
-        fi
-        [[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
-    fi
-}
-`
-)
-
 // root help command
 var helpCommand = &cobra.Command{
 	Use:   "help",

cmd/info/info.go

@@ -21,22 +21,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-type position int
-
-const (
-	positionMiddle position = 1 << iota
-	positionLeft
-	positionRight
-	positionNone position = 0
-	positionAll  position = positionRight<<1 - 1
-)
-
 var (
 	checkNormalization bool
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
-	positionList       = []position{positionMiddle, positionLeft, positionRight}
 )
 
 func init() {
@@ -70,7 +59,7 @@ a bit of go code for each one.
 type results struct {
 	f                    fs.Fs
 	mu                   sync.Mutex
-	stringNeedsEscaping  map[string]position
+	charNeedsEscaping    map[rune]bool
 	maxFileLength        int
 	canWriteUnnormalized bool
 	canReadUnnormalized  bool
@@ -80,8 +69,8 @@ type results struct {
 
 func newResults(f fs.Fs) *results {
 	return &results{
-		f:                   f,
-		stringNeedsEscaping: make(map[string]position),
+		f:                 f,
+		charNeedsEscaping: make(map[rune]bool),
 	}
 }
 
@@ -90,13 +79,13 @@ func (r *results) Print() {
 	fmt.Printf("// %s\n", r.f.Name())
 	if checkControl {
 		escape := []string{}
-		for c, needsEscape := range r.stringNeedsEscaping {
-			if needsEscape != positionNone {
+		for c, needsEscape := range r.charNeedsEscaping {
+			if needsEscape {
 				escape = append(escape, fmt.Sprintf("0x%02X", c))
 			}
 		}
 		sort.Strings(escape)
-		fmt.Printf("stringNeedsEscaping = []byte{\n")
+		fmt.Printf("charNeedsEscaping = []byte{\n")
 		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
 		fmt.Printf("}\n")
 	}
@@ -141,45 +130,20 @@ func (r *results) checkUTF8Normalization() {
 	}
 }
 
-func (r *results) checkStringPositions(s string) {
-	fs.Infof(r.f, "Writing position file 0x%0X", s)
-	positionError := positionNone
-
-	for _, pos := range positionList {
-		path := ""
-		switch pos {
-		case positionMiddle:
-			path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
-		case positionLeft:
-			path = fmt.Sprintf("%s-position-left-%0X", s, s)
-		case positionRight:
-			path = fmt.Sprintf("position-right-%0X-%s", s, s)
-		default:
-			panic("invalid position: " + pos.String())
-		}
-		_, writeErr := r.writeFile(path)
-		if writeErr != nil {
-			fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
-		} else {
-			fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
-		}
-		obj, getErr := r.f.NewObject(path)
-		if getErr != nil {
-			fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
-		} else {
-			if obj.Size() != 50 {
-				fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
-			} else {
-				fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
-			}
-		}
-		if writeErr != nil || getErr != nil {
-			positionError += pos
-		}
+// check we can write file with the rune passed in
+func (r *results) checkChar(c rune) {
+	fs.Infof(r.f, "Writing file 0x%02X", c)
+	path := fmt.Sprintf("0x%02X-%c-", c, c)
+	_, err := r.writeFile(path)
+	escape := false
+	if err != nil {
+		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
+		escape = true
+	} else {
+		fs.Infof(r.f, "OK writing file 0x%02X", c)
 	}
-
 	r.mu.Lock()
-	r.stringNeedsEscaping[s] = positionError
+	r.charNeedsEscaping[c] = escape
 	r.mu.Unlock()
 }
 
@@ -193,28 +157,19 @@ func (r *results) checkControls() {
 	}
 	var wg sync.WaitGroup
 	for i := rune(0); i < 128; i++ {
-		s := string(i)
 		if i == 0 || i == '/' {
 			// We're not even going to check NULL or /
-			r.stringNeedsEscaping[s] = positionAll
+			r.charNeedsEscaping[i] = true
 			continue
 		}
 		wg.Add(1)
-		go func(s string) {
+		c := i
+		go func() {
 			defer wg.Done()
 			token := <-tokens
-			r.checkStringPositions(s)
+			r.checkChar(c)
 			tokens <- token
-		}(s)
-	}
-	for _, s := range []string{"＼", "\xBF", "\xFE"} {
-		wg.Add(1)
-		go func(s string) {
-			defer wg.Done()
-			token := <-tokens
-			r.checkStringPositions(s)
-			tokens <- token
-		}(s)
+		}()
 	}
 	wg.Wait()
 	fs.Infof(r.f, "Done trying to create control character file names")
@@ -313,35 +268,3 @@ func readInfo(f fs.Fs) error {
 	r.Print()
 	return nil
 }
-
-func (e position) String() string {
-	switch e {
-	case positionNone:
-		return "none"
-	case positionAll:
-		return "all"
-	}
-	var buf bytes.Buffer
-	if e&positionMiddle != 0 {
-		buf.WriteString("middle")
-		e &= ^positionMiddle
-	}
-	if e&positionLeft != 0 {
-		if buf.Len() != 0 {
-			buf.WriteRune(',')
-		}
-		buf.WriteString("left")
-		e &= ^positionLeft
-	}
-	if e&positionRight != 0 {
-		if buf.Len() != 0 {
-			buf.WriteRune(',')
-		}
-		buf.WriteString("right")
-		e &= ^positionRight
-	}
-	if e != positionNone {
-		panic("invalid position")
-	}
-	return buf.String()
-}

cmd/info/process.sh

@@ -1,40 +0,0 @@
-set -euo pipefail
-
-for f in info-*.log; do 
-  for pos in middle left right; do
-    egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos
-    egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos
-  done
-  {
-    echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}"
-    echo "Write\tWrite\tWrite\tGet\tGet\tGet"
-    echo "Mid\tLeft\tRight\tMid\tLeft\tRight"
-    paste $f.write_{middle,left,right} $f.get_{middle,left,right}
-  } > $f.csv
-done
-
-for f in info-*.list; do
-  for pos in middle left right; do
-    cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos
-  done
-  {
-    echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}"
-    echo "List\tList\tList"
-    echo "Mid\tLeft\tRight"
-    for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do
-      echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss)
-      echo -n "\t"
-      echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/'    $f.left   | tr -d "\t\r" | grep -a . || echo Miss)
-      echo -n "\t"
-      echo    $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/'   $f.right  | tr -d "\t\r" | grep -a . || echo Miss)
-      # echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t"
-      # echo -n $(grep -a "position-left-$e"    $f.left   | tr -d "\t\r" || echo Miss)"\t"
-      # echo    $(grep -a "position-right-$e-"  $f.right  | tr -d "\t\r" || echo Miss)
-    done
-    } > $f.csv
-done
-
-for f in info-*.list; do 
-    paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv
-done
-paste *.full.csv > info-complete.csv

cmd/info/test.cmd

@@ -1,3 +0,0 @@
-rclone.exe purge    info
-rclone.exe info -vv info > info-LocalWindows.log  2>&1
-rclone.exe ls   -vv info > info-LocalWindows.list 2>&1

cmd/info/test.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env zsh
-#
-# example usage: 
-# $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh --list | \
-#   parallel -P20 $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh
-
-export PATH=$GOPATH/src/github.com/ncw/rclone:$PATH
-
-typeset -A allRemotes
- allRemotes=(
-  TestAmazonCloudDrive '--low-level-retries=2 --checkers=5'
-  TestB2 ''
-  TestBox ''
-  TestDrive '--tpslimit=5'
-  TestCrypt ''
-  TestDropbox '--checkers=1'
-  TestJottacloud ''
-  TestMega ''
-  TestOneDrive ''
-  TestOpenDrive '--low-level-retries=2 --checkers=5'
-  TestPcloud '--low-level-retries=2 --timeout=15s'
-  TestS3 ''
-  Local ''
-)
-
-set -euo pipefail
-
-if [[ $# -eq 0 ]]; then
- set -- ${(k)allRemotes[@]}
-elif [[ $1 = --list ]]; then
-  printf '%s\n' ${(k)allRemotes[@]}
-  exit 0
-fi
-
-for remote; do
-  dir=$remote:infotest
-  if [[ $remote = Local ]]; then
-    dir=infotest
-  fi
-  rclone purge    $dir || :
-  rclone info -vv $dir ${=allRemotes[$remote]} &> info-$remote.log
-  rclone ls   -vv $dir &> info-$remote.list
-done

cmd/listremotes/listremotes.go

@@ -16,7 +16,7 @@ var (
 
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
-	commandDefintion.Flags().BoolVarP(&listLong, "long", "", listLong, "Show the type as well as names.")
+	commandDefintion.Flags().BoolVarP(&listLong, "long", "l", listLong, "Show the type as well as names.")
 }
 
 var commandDefintion = &cobra.Command{

cmd/lsf/lsf.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/operations"
+	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -66,10 +67,8 @@ output:
     s - size
     t - modification time
     h - hash
-    i - ID of object
-    o - Original ID of underlying object
+    i - ID of object if known
     m - MimeType of object if known
-    e - encrypted name
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -162,10 +161,6 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 	list.SetCSV(csv)
 	list.SetDirSlash(dirSlash)
 	list.SetAbsolute(absolute)
-	var opt = operations.ListJSONOpt{
-		NoModTime: true,
-		Recurse:   recurse,
-	}
 
 	for _, char := range format {
 		switch char {
@@ -173,38 +168,38 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 			list.AddPath()
 		case 't':
 			list.AddModTime()
-			opt.NoModTime = false
 		case 's':
 			list.AddSize()
 		case 'h':
 			list.AddHash(hashType)
-			opt.ShowHash = true
 		case 'i':
 			list.AddID()
 		case 'm':
 			list.AddMimeType()
-		case 'e':
-			list.AddEncrypted()
-			opt.ShowEncrypted = true
-		case 'o':
-			list.AddOrigID()
-			opt.ShowOrigIDs = true
 		default:
 			return errors.Errorf("Unknown format character %q", char)
 		}
 	}
 
-	return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
-		if item.IsDir {
-			if filesOnly {
-				return nil
-			}
-		} else {
-			if dirsOnly {
-				return nil
-			}
+	return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error {
+		if err != nil {
+			fs.CountError(err)
+			fs.Errorf(path, "error listing: %v", err)
+			return nil
+		}
+		for _, entry := range entries {
+			_, isDir := entry.(fs.Directory)
+			if isDir {
+				if filesOnly {
+					continue
+				}
+			} else {
+				if dirsOnly {
+					continue
+				}
+			}
+			_, _ = fmt.Fprintln(out, list.Format(entry))
 		}
-		_, _ = fmt.Fprintln(out, list.Format(item))
 		return nil
 	})
 }

cmd/lsjson/lsjson.go

@@ -60,13 +60,7 @@ If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
 will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
 When used without --recursive the Path will always be the same as Name.
 
-The time is in RFC3339 format with up to nanosecond precision.  The
-number of decimal digits in the seconds will depend on the precision
-that the remote can hold the times, so if times are accurate to the
-nearest millisecond (eg Google Drive) then 3 digits will always be
-shown ("2017-05-31T16:15:57.034+01:00") whereas if the times are
-accurate to the nearest second (Dropbox, Box, WebDav etc) no digits
-will be shown ("2017-05-31T16:15:57+01:00").
+The time is in RFC3339 format with nanosecond precision.
 
 The whole output can be processed as a JSON blob, or alternatively it
 can be processed line by line as each item is written one to a line.

cmd/mount/dir.go

@@ -37,8 +37,6 @@ func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
 	a.Crtime = modTime
 	// FIXME include Valid so get some caching?
 	// FIXME fs.Debugf(d.path, "Dir.Attr %+v", a)
-	a.Size = 512
-	a.Blocks = 1
 	return nil
 }
 

cmd/mount/handle.go

@@ -45,7 +45,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	if err != nil {
 		return translateError(err)
 	}
-	resp.Size = n
+	resp.Size = int(n)
 	return nil
 }
 

cmd/mount/test/seek_speed.go

@@ -20,12 +20,12 @@ var (
 )
 
 func randomSeekTest(size int64, in *os.File, name string) {
-	startTime := time.Now()
 	start := rand.Int63n(size)
 	blockSize := rand.Intn(*maxBlockSize)
 	if int64(blockSize) > size-start {
 		blockSize = int(size - start)
 	}
+	log.Printf("Reading %d from %d", blockSize, start)
 
 	_, err := in.Seek(start, io.SeekStart)
 	if err != nil {
@@ -37,8 +37,6 @@ func randomSeekTest(size int64, in *os.File, name string) {
 	if err != nil {
 		log.Fatalf("Read failed on %q: %v", name, err)
 	}
-
-	log.Printf("Reading %d from %d took %v ", blockSize, start, time.Since(startTime))
 }
 
 func main() {
@@ -50,12 +48,10 @@ func main() {
 	rand.Seed(*randSeed)
 
 	name := args[0]
-	openStart := time.Now()
 	in, err := os.Open(name)
 	if err != nil {
 		log.Fatalf("Couldn't open %q: %v", name, err)
 	}
-	log.Printf("File Open took %v", time.Since(openStart))
 
 	fi, err := in.Stat()
 	if err != nil {

cmd/mountlib/mount.go

@@ -326,11 +326,7 @@ func ClipBlocks(b *uint64) {
 	var max uint64
 	switch runtime.GOOS {
 	case "windows":
-		if runtime.GOARCH == "386" {
-			max = (1 << 32) - 1
-		} else {
-			max = (1 << 43) - 1
-		}
+		max = (1 << 43) - 1
 	case "darwin":
 		// OSX FUSE only supports 32 bit number of blocks
 		// https://github.com/osxfuse/osxfuse/issues/396

cmd/move/move.go

@@ -10,13 +10,11 @@ import (
 // Globals
 var (
 	deleteEmptySrcDirs = false
-	createEmptySrcDirs = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
 	commandDefintion.Flags().BoolVarP(&deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move")
-	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move")
 }
 
 var commandDefintion = &cobra.Command{
@@ -54,7 +52,7 @@ can speed transfers up greatly.
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
+				return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs)
 			}
 			return operations.MoveFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/moveto/moveto.go

@@ -52,7 +52,7 @@ transfer.
 
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.MoveDir(fdst, fsrc, false, false)
+				return sync.MoveDir(fdst, fsrc, false)
 			}
 			return operations.MoveFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/ncdu/ncdu.go

@@ -10,7 +10,6 @@ import (
 	"sort"
 	"strings"
 
-	runewidth "github.com/mattn/go-runewidth"
 	"github.com/ncw/rclone/cmd"
 	"github.com/ncw/rclone/cmd/ncdu/scan"
 	"github.com/ncw/rclone/fs"
@@ -123,7 +122,7 @@ func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface
 func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
 	for _, c := range msg {
 		termbox.SetCell(x, y, c, fg, bg)
-		x += runewidth.RuneWidth(c)
+		x++
 		if x >= xmax {
 			return
 		}

cmd/rcd/rcd.go

@@ -17,7 +17,7 @@ var commandDefintion = &cobra.Command{
 	Use:   "rcd <path to files to serve>*",
 	Short: `Run rclone listening to remote control commands only.`,
 	Long: `
-This runs rclone so that it only listens to remote control commands.
+This runs rclone so that it only listents to remote control commands.
 
 This is useful if you are controlling rclone via the rc API.
 

cmd/serve/dlna/cds.go

@@ -158,7 +158,7 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 		}, nil
 	case "Browse":
 		var browse browse
-		if err := xml.Unmarshal(argsXML, &browse); err != nil {
+		if err := xml.Unmarshal([]byte(argsXML), &browse); err != nil {
 			return nil, err
 		}
 		obj, err := cds.objectFromID(browse.ObjectID)
@@ -179,7 +179,7 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 				}
 				return
 			}():]
-			if browse.RequestedCount != 0 && browse.RequestedCount < len(objs) {
+			if browse.RequestedCount != 0 && int(browse.RequestedCount) < len(objs) {
 				objs = objs[:browse.RequestedCount]
 			}
 			result, err := xml.Marshal(objs)

cmd/serve/dlna/cm-service-desc.go

@@ -1,184 +0,0 @@
-package dlna
-
-const connectionManagerServiceDescription = `<?xml version="1.0" encoding="UTF-8"?>
-<scpd xmlns="urn:schemas-upnp-org:service-1-0">
-  <specVersion>
-    <major>1</major>
-    <minor>0</minor>
-  </specVersion>
-  <actionList>
-    <action>
-      <name>GetProtocolInfo</name>
-      <argumentList>
-        <argument>
-          <name>Source</name>
-          <direction>out</direction>
-          <relatedStateVariable>SourceProtocolInfo</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Sink</name>
-          <direction>out</direction>
-          <relatedStateVariable>SinkProtocolInfo</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>PrepareForConnection</name>
-      <argumentList>
-        <argument>
-          <name>RemoteProtocolInfo</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>PeerConnectionManager</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>PeerConnectionID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Direction</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ConnectionID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>AVTransportID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RcsID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ConnectionComplete</name>
-      <argumentList>
-        <argument>
-          <name>ConnectionID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetCurrentConnectionIDs</name>
-      <argumentList>
-        <argument>
-          <name>ConnectionIDs</name>
-          <direction>out</direction>
-          <relatedStateVariable>CurrentConnectionIDs</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetCurrentConnectionInfo</name>
-      <argumentList>
-        <argument>
-          <name>ConnectionID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RcsID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>AVTransportID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ProtocolInfo</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>PeerConnectionManager</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>PeerConnectionID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Direction</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Status</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ConnectionStatus</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-  </actionList>
-  <serviceStateTable>
-    <stateVariable sendEvents="yes">
-      <name>SourceProtocolInfo</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>SinkProtocolInfo</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>CurrentConnectionIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ConnectionStatus</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>OK</allowedValue>
-        <allowedValue>ContentFormatMismatch</allowedValue>
-        <allowedValue>InsufficientBandwidth</allowedValue>
-        <allowedValue>UnreliableChannel</allowedValue>
-        <allowedValue>Unknown</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ConnectionManager</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Direction</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>Input</allowedValue>
-        <allowedValue>Output</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ProtocolInfo</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ConnectionID</name>
-      <dataType>i4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_AVTransportID</name>
-      <dataType>i4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_RcsID</name>
-      <dataType>i4</dataType>
-    </stateVariable>
-  </serviceStateTable>
-</scpd>`

cmd/serve/dlna/dlna.go

@@ -84,21 +84,6 @@ var services = []*service{
 		},
 		SCPD: contentDirectoryServiceDescription,
 	},
-	{
-		Service: upnp.Service{
-			ServiceType: "urn:schemas-upnp-org:service:ConnectionManager:1",
-			ServiceId:   "urn:upnp-org:serviceId:ConnectionManager",
-			ControlURL:  serviceControlURL,
-		},
-		SCPD: connectionManagerServiceDescription,
-	},
-}
-
-func init() {
-	for _, s := range services {
-		p := path.Join("/scpd", s.ServiceId)
-		s.SCPDURL = p
-	}
 }
 
 func devices() []string {
@@ -265,6 +250,9 @@ func (s *server) initMux(mux *http.ServeMux) {
 
 	// Install handlers to serve SCPD for each UPnP service.
 	for _, s := range services {
+		p := path.Join("/scpd", s.ServiceId)
+		s.SCPDURL = p
+
 		mux.HandleFunc(s.SCPDURL, func(serviceDesc string) http.HandlerFunc {
 			return func(w http.ResponseWriter, r *http.Request) {
 				w.Header().Set("content-type", `text/xml; charset="utf-8"`)

cmd/serve/dlna/dlna_test.go

@@ -59,11 +59,6 @@ func TestRootSCPD(t *testing.T) {
 	// Make sure that the SCPD contains a CDS service.
 	require.Contains(t, string(body),
 		"<serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>")
-	// Make sure that the SCPD contains a CM service.
-	require.Contains(t, string(body),
-		"<serviceType>urn:schemas-upnp-org:service:ConnectionManager:1</serviceType>")
-	// Ensure that the SCPD url is configured.
-	require.Regexp(t, "<SCPDURL>/.*</SCPDURL>", string(body))
 }
 
 // Make sure that it serves content from the remote.

cmd/serve/restic/restic.go

@@ -330,12 +330,25 @@ func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote stri
 	ls := listItems{}
 
 	// if remote supports ListR use that directly, otherwise use recursive Walk
-	err := walk.ListR(s.f, remote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
-		for _, entry := range entries {
-			ls.add(entry)
-		}
-		return nil
-	})
+	var err error
+	if ListR := s.f.Features().ListR; ListR != nil {
+		err = ListR(remote, func(entries fs.DirEntries) error {
+			for _, entry := range entries {
+				ls.add(entry)
+			}
+			return nil
+		})
+	} else {
+		err = walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error {
+			if err == nil {
+				for _, entry := range entries {
+					ls.add(entry)
+				}
+			}
+			return err
+		})
+	}
+
 	if err != nil {
 		_, err = fserrors.Cause(err)
 		if err != fs.ErrorDirNotFound {

cmd/sync/sync.go

@@ -6,13 +6,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var (
-	createEmptySrcDirs = false
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
-	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync")
 }
 
 var commandDefintion = &cobra.Command{
@@ -44,7 +39,7 @@ go there.
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, fdst := cmd.NewFsSrcDst(args)
 		cmd.Run(true, true, command, func() error {
-			return sync.Sync(fdst, fsrc, createEmptySrcDirs)
+			return sync.Sync(fdst, fsrc)
 		})
 	},
 }

docs/content/about.md

@@ -29,7 +29,6 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
 * {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
 * {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
-* {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
 * {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}}
 * {{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}}
 * {{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}}
@@ -45,7 +44,6 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="put.io" home="https://put.io/" config="/webdav/#put-io" >}}
 * {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
 * {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
-* {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}
 * {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SFTP" config="/sftp/" >}}
 * {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
 * {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}

docs/content/authors.md

@@ -228,23 +228,3 @@ Contributors
   * nicolov <nicolov@users.noreply.github.com>
   * Dario Guzik <dario@guzik.com.ar>
   * qip <qip@users.noreply.github.com>
-  * yair@unicorn <yair@unicorn>
-  * Matt Robinson <brimstone@the.narro.ws>
-  * kayrus <kay.diam@gmail.com>
-  * Rémy Léone <remy.leone@gmail.com>
-  * Wojciech Smigielski <wojciech.hieronim.smigielski@gmail.com>
-  * weetmuts <oehrstroem@gmail.com>
-  * Jonathan <vanillajonathan@users.noreply.github.com>
-  * James Carpenter <orbsmiv@users.noreply.github.com>
-  * Vince <vince0villamora@gmail.com>
-  * Nestar47 <47841759+Nestar47@users.noreply.github.com>
-  * Six <brbsix@gmail.com>
-  * Alexandru Bumbacea <alexandru.bumbacea@booking.com>
-  * calisro <robert.calistri@gmail.com>
-  * Dr.Rx <david.rey@nventive.com>
-  * marcintustin <marcintustin@users.noreply.github.com>
-  * jaKa Močnik <jaka@koofr.net>
-  * Fionera <fionera@fionera.de>
-  * Dan Walters <dan@walters.io>
-  * Danil Semelenov <sgtpep@users.noreply.github.com>
-  * xopez <28950736+xopez@users.noreply.github.com>

docs/content/b2.md

@@ -16,11 +16,9 @@ Here is an example of making a b2 configuration.  First run
 
     rclone config
 
-This will guide you through an interactive setup process.  To authenticate
-you will either need your Account ID (a short hex number) and Master
-Application Key (a long hex number) OR an Application Key, which is the
-recommended method. See below for further details on generating and using
-an Application Key.
+This will guide you through an interactive setup process.  You will
+need your account number (a short hex number) and key (a long hex
+number) which you can get from the b2 control panel.
 
 ```
 No remotes found - make a new one
@@ -104,10 +102,10 @@ You can use these with rclone too; you will need to use rclone version 1.43
 or later.
 
 Follow Backblaze's docs to create an Application Key with the required
-permission and add the `applicationKeyId` as the `account` and the
+permission and add the `Application Key ID` as the `account` and the
 `Application Key` itself as the `key`.
 
-Note that you must put the _applicationKeyId_ as the `account` – you
+Note that you must put the Application Key ID as the `account` - you
 can't use the master Account ID.  If you try then B2 will return 401
 errors.
 
@@ -393,21 +391,12 @@ Upload chunk size. Must fit in memory.
 When uploading large files, chunk the file into this size.  Note that
 these chunks are buffered in memory and there might a maximum of
 "--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimum size.
+minimim size.
 
 - Config:      chunk_size
 - Env Var:     RCLONE_B2_CHUNK_SIZE
 - Type:        SizeSuffix
 - Default:     96M
 
-#### --b2-disable-checksum
-
-Disable checksums for large (> upload cutoff) files
-
-- Config:      disable_checksum
-- Env Var:     RCLONE_B2_DISABLE_CHECKSUM
-- Type:        bool
-- Default:     false
-
 <!--- autogenerated options stop -->
 

docs/content/box.md

@@ -112,17 +112,6 @@ To copy a local directory to an Box directory called backup
 
     rclone copy /home/source remote:backup
 
-### Using rclone with an Enterprise account with SSO ###
-
-If you have an "Enterprise" account type with Box with single sign on
-(SSO), you need to create a password to use Box with rclone. This can
-be done at your Enterprise Box account by going to Settings, "Account"
-Tab, and then set the password in the "Authentication" field.
-
-Once you have done this, you can setup your Enterprise Box account
-using the same procedure detailed above in the, using the password you
-have just set.
-
 ### Invalid refresh token ###
 
 According to the [box docs](https://developer.box.com/v2.0/docs/oauth-20#section-6-using-the-access-and-refresh-tokens):

docs/content/changelog.md

@@ -1,140 +1,11 @@
 ---
 title: "Documentation"
 description: "Rclone Changelog"
-date: "2019-02-09"
+date: "2018-11-24"
 ---
 
 # Changelog
 
-## v1.46 - 2019-02-09
-
-* New backends
-    * Support Alibaba Cloud (Aliyun) OSS via the s3 backend (Nick Craig-Wood)
-* New commands
-    * serve dlna: serves a remove via DLNA for the local network (nicolov)
-* New Features
-    * copy, move: Restore deprecated `--no-traverse` flag (Nick Craig-Wood)
-        * This is useful for when transferring a small number of files into a large destination
-    * genautocomplete: Add remote path completion for bash completion (Christopher Peterson & Danil Semelenov)
-    * Buffer memory handling reworked to return memory to the OS better (Nick Craig-Wood)
-        * Buffer recycling library to replace sync.Pool
-        * Optionally use memory mapped memory for better memory shrinking
-        * Enable with `--use-mmap` if having memory problems - not default yet
-    * Parallelise reading of files specified by `--files-from` (Nick Craig-Wood)
-    * check: Add stats showing total files matched. (Dario Guzik)
-    * Allow rename/delete open files under Windows (Nick Craig-Wood)
-    * lsjson: Use exactly the correct number of decimal places in the seconds (Nick Craig-Wood)
-    * Add cookie support with cmdline switch `--use-cookies` for all HTTP based remotes (qip)
-    * Warn if `--checksum` is set but there are no hashes available (Nick Craig-Wood)
-    * Rework rate limiting (pacer) to be more accurate and allow bursting (Nick Craig-Wood)
-    * Improve error reporting for too many/few arguments in commands (Nick Craig-Wood)
-    * listremotes: Remove `-l` short flag as it conflicts with the new global flag (weetmuts)
-    * Make http serving with auth generate INFO messages on auth fail (Nick Craig-Wood)
-* Bug Fixes
-    * Fix layout of stats (Nick Craig-Wood)
-    * Fix `--progress` crash under Windows Jenkins (Nick Craig-Wood)
-    * Fix transfer of google/onedrive docs by calling Rcat in Copy when size is -1 (Cnly)
-    * copyurl: Fix checking of `--dry-run` (Denis Skovpen)
-* Mount
-    * Check that mountpoint and local directory to mount don't overlap (Nick Craig-Wood)
-    * Fix mount size under 32 bit Windows (Nick Craig-Wood)
-* VFS
-    * Implement renaming of directories for backends without DirMove (Nick Craig-Wood)
-        * now all backends except b2 support renaming directories
-    * Implement `--vfs-cache-max-size` to limit the total size of the cache (Nick Craig-Wood)
-    * Add `--dir-perms` and `--file-perms` flags to set default permissions (Nick Craig-Wood)
-    * Fix deadlock on concurrent operations on a directory (Nick Craig-Wood)
-    * Fix deadlock between RWFileHandle.close and File.Remove (Nick Craig-Wood)
-    * Fix renaming/deleting open files with cache mode "writes" under Windows (Nick Craig-Wood)
-    * Fix panic on rename with `--dry-run` set (Nick Craig-Wood)
-    * Fix vfs/refresh with recurse=true needing the `--fast-list` flag
-* Local
-    * Add support for `-l`/`--links` (symbolic link translation) (yair@unicorn)
-        * this works by showing links as `link.rclonelink` - see local backend docs for more info
-        * this errors if used with `-L`/`--copy-links`
-    * Fix renaming/deleting open files on Windows (Nick Craig-Wood)
-* Crypt
-    * Check for maximum length before decrypting filename to fix panic (Garry McNulty)
-* Azure Blob
-    * Allow building azureblob backend on *BSD (themylogin)
-    * Use the rclone HTTP client to support `--dump headers`, `--tpslimit` etc (Nick Craig-Wood)
-    * Use the s3 pacer for 0 delay in non error conditions (Nick Craig-Wood)
-    * Ignore directory markers (Nick Craig-Wood)
-    * Stop Mkdir attempting to create existing containers (Nick Craig-Wood)
-* B2
-    * cleanup: will remove unfinished large files >24hrs old (Garry McNulty)
-    * For a bucket limited application key check the bucket name (Nick Craig-Wood)
-        * before this, rclone would use the authorised bucket regardless of what you put on the command line
-    * Added `--b2-disable-checksum` flag (Wojciech Smigielski)
-        * this enables large files to be uploaded without a SHA-1 hash for speed reasons
-* Drive
-    * Set default pacer to 100ms for 10 tps (Nick Craig-Wood)
-        * This fits the Google defaults much better and reduces the 403 errors massively
-        * Add `--drive-pacer-min-sleep` and `--drive-pacer-burst` to control the pacer
-    * Improve ChangeNotify support for items with multiple parents (Fabian Möller)
-    * Fix ListR for items with multiple parents - this fixes oddities with `vfs/refresh` (Fabian Möller)
-    * Fix using `--drive-impersonate` and appfolders (Nick Craig-Wood)
-    * Fix google docs in rclone mount for some (not all) applications (Nick Craig-Wood)
-* Dropbox
-    * Retry-After support for Dropbox backend (Mathieu Carbou)
-* FTP
-    * Wait for 60 seconds for a connection to Close then declare it dead (Nick Craig-Wood)
-        * helps with indefinite hangs on some FTP servers
-* Google Cloud Storage
-    * Update google cloud storage endpoints (weetmuts)
-* HTTP
-    * Add an example with username and password which is supported but wasn't documented (Nick Craig-Wood)
-    * Fix backend with `--files-from` and non-existent files (Nick Craig-Wood)
-* Hubic
-    * Make error message more informative if authentication fails (Nick Craig-Wood)
-* Jottacloud
-    * Resume and deduplication support (Oliver Heyme)
-    * Use token auth for all API requests Don't store password anymore (Sebastian Bünger)
-    * Add support for 2-factor authentification (Sebastian Bünger)
-* Mega
-    * Implement v2 account login which fixes logins for newer Mega accounts (Nick Craig-Wood)
-    * Return error if an unknown length file is attempted to be uploaded (Nick Craig-Wood)
-    * Add new error codes for better error reporting (Nick Craig-Wood)
-* Onedrive
-    * Fix broken support for "shared with me" folders (Alex Chen)
-    * Fix root ID not normalised (Cnly)
-    * Return err instead of panic on unknown-sized uploads (Cnly)
-* Qingstor
-    * Fix go routine leak on multipart upload errors (Nick Craig-Wood)
-    * Add upload chunk size/concurrency/cutoff control (Nick Craig-Wood)
-    * Default `--qingstor-upload-concurrency` to 1 to work around bug (Nick Craig-Wood)
-* S3
-    * Implement `--s3-upload-cutoff` for single part uploads below this (Nick Craig-Wood)
-    * Change `--s3-upload-concurrency` default to 4 to increase perfomance (Nick Craig-Wood)
-    * Add `--s3-bucket-acl` to control bucket ACL (Nick Craig-Wood)
-    * Auto detect region for buckets on operation failure (Nick Craig-Wood)
-    * Add GLACIER storage class (William Cocker)
-    * Add Scaleway to s3 documentation (Rémy Léone)
-    * Add AWS endpoint eu-north-1 (weetmuts)
-* SFTP
-    * Add support for PEM encrypted private keys (Fabian Möller)
-    * Add option to force the usage of an ssh-agent (Fabian Möller)
-    * Perform environment variable expansion on key-file (Fabian Möller)
-    * Fix rmdir on Windows based servers (eg CrushFTP) (Nick Craig-Wood)
-    * Fix rmdir deleting directory contents on some SFTP servers (Nick Craig-Wood)
-    * Fix error on dangling symlinks (Nick Craig-Wood)
-* Swift
-    * Add `--swift-no-chunk` to disable segmented uploads in rcat/mount (Nick Craig-Wood)
-    * Introduce application credential auth support (kayrus)
-    * Fix memory usage by slimming Object (Nick Craig-Wood)
-    * Fix extra requests on upload (Nick Craig-Wood)
-    * Fix reauth on big files (Nick Craig-Wood)
-* Union
-    * Fix poll-interval not working (Nick Craig-Wood)
-* WebDAV
-    * Support About which means rclone mount will show the correct disk size (Nick Craig-Wood)
-    * Support MD5 and SHA1 hashes with Owncloud and Nextcloud (Nick Craig-Wood)
-    * Fail soft on time parsing errors (Nick Craig-Wood)
-    * Fix infinite loop on failed directory creation (Nick Craig-Wood)
-    * Fix identification of directories for Bitrix Site Manager (Nick Craig-Wood)
-    * Fix upload of 0 length files on some servers (Nick Craig-Wood)
-    * Fix if MKCOL fails with 423 Locked assume the directory exists (Nick Craig-Wood)
-
 ## v1.45 - 2018-11-24
 
 * New backends

docs/content/commands/rclone.md

@@ -1,5 +1,5 @@
 ---
-date: 2019-02-09T10:42:18Z
+date: 2018-11-24T13:43:29Z
 title: "rclone"
 slug: rclone
 url: /commands/rclone/
@@ -26,301 +26,283 @@ rclone [flags]
 ### Options
 
 ```
-      --acd-auth-url string                          Auth server URL.
-      --acd-client-id string                         Amazon Application Client ID.
-      --acd-client-secret string                     Amazon Application Client Secret.
-      --acd-templink-threshold SizeSuffix            Files >= this size will be downloaded via their tempLink. (default 9G)
-      --acd-token-url string                         Token server url.
-      --acd-upload-wait-per-gb Duration              Additional time per GB to wait after a failed complete upload to see if it appears. (default 3m0s)
-      --alias-remote string                          Remote or path to alias.
-      --ask-password                                 Allow prompt for password for encrypted configuration. (default true)
-      --auto-confirm                                 If enabled, do not request console confirmation.
-      --azureblob-access-tier string                 Access tier of blob: hot, cool or archive.
-      --azureblob-account string                     Storage Account Name (leave blank to use connection string or SAS URL)
-      --azureblob-chunk-size SizeSuffix              Upload chunk size (<= 100MB). (default 4M)
-      --azureblob-endpoint string                    Endpoint for the service
-      --azureblob-key string                         Storage Account Key (leave blank to use connection string or SAS URL)
-      --azureblob-list-chunk int                     Size of blob list. (default 5000)
-      --azureblob-sas-url string                     SAS URL for container level access only
-      --azureblob-upload-cutoff SizeSuffix           Cutoff for switching to chunked upload (<= 256MB). (default 256M)
-      --b2-account string                            Account ID or Application Key ID
-      --b2-chunk-size SizeSuffix                     Upload chunk size. Must fit in memory. (default 96M)
-      --b2-disable-checksum                          Disable checksums for large (> upload cutoff) files
-      --b2-endpoint string                           Endpoint for the service.
-      --b2-hard-delete                               Permanently delete files on remote removal, otherwise hide files.
-      --b2-key string                                Application Key
-      --b2-test-mode string                          A flag string for X-Bz-Test-Mode header for debugging.
-      --b2-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload. (default 200M)
-      --b2-versions                                  Include old versions in directory listings.
-      --backup-dir string                            Make backups into hierarchy based in DIR.
-      --bind string                                  Local address to bind to for outgoing connections, IPv4, IPv6 or name.
-      --box-client-id string                         Box App Client Id.
-      --box-client-secret string                     Box App Client Secret
-      --box-commit-retries int                       Max number of times to try committing a multipart file. (default 100)
-      --box-upload-cutoff SizeSuffix                 Cutoff for switching to multipart upload (>= 50MB). (default 50M)
-      --buffer-size SizeSuffix                       In memory buffer size when reading files for each --transfer. (default 16M)
-      --bwlimit BwTimetable                          Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.
-      --cache-chunk-clean-interval Duration          How often should the cache perform cleanups of the chunk storage. (default 1m0s)
-      --cache-chunk-no-memory                        Disable the in-memory cache for storing chunks during streaming.
-      --cache-chunk-path string                      Directory to cache chunk files. (default "$HOME/.cache/rclone/cache-backend")
-      --cache-chunk-size SizeSuffix                  The size of a chunk (partial file data). (default 5M)
-      --cache-chunk-total-size SizeSuffix            The total size that the chunks can take up on the local disk. (default 10G)
-      --cache-db-path string                         Directory to store file structure metadata DB. (default "$HOME/.cache/rclone/cache-backend")
-      --cache-db-purge                               Clear all the cached data for this remote on start.
-      --cache-db-wait-time Duration                  How long to wait for the DB to be available - 0 is unlimited (default 1s)
-      --cache-dir string                             Directory rclone will use for caching. (default "$HOME/.cache/rclone")
-      --cache-info-age Duration                      How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
-      --cache-plex-insecure string                   Skip all certificate verifications when connecting to the Plex server
-      --cache-plex-password string                   The password of the Plex user
-      --cache-plex-url string                        The URL of the Plex server
-      --cache-plex-username string                   The username of the Plex user
-      --cache-read-retries int                       How many times to retry a read from a cache storage. (default 10)
-      --cache-remote string                          Remote to cache.
-      --cache-rps int                                Limits the number of requests per second to the source FS (-1 to disable) (default -1)
-      --cache-tmp-upload-path string                 Directory to keep temporary files until they are uploaded.
-      --cache-tmp-wait-time Duration                 How long should files be stored in local cache before being uploaded (default 15s)
-      --cache-workers int                            How many workers should run in parallel to download chunks. (default 4)
-      --cache-writes                                 Cache file data on writes through the FS
-      --checkers int                                 Number of checkers to run in parallel. (default 8)
-  -c, --checksum                                     Skip based on checksum (if available) & size, not mod-time & size
-      --config string                                Config file. (default "/home/ncw/.rclone.conf")
-      --contimeout duration                          Connect timeout (default 1m0s)
-  -L, --copy-links                                   Follow symlinks and copy the pointed to item.
-      --cpuprofile string                            Write cpu profile to file
-      --crypt-directory-name-encryption              Option to either encrypt directory names or leave them intact. (default true)
-      --crypt-filename-encryption string             How to encrypt the filenames. (default "standard")
-      --crypt-password string                        Password or pass phrase for encryption.
-      --crypt-password2 string                       Password or pass phrase for salt. Optional but recommended.
-      --crypt-remote string                          Remote to encrypt/decrypt.
-      --crypt-show-mapping                           For all files listed show how the names encrypt.
-      --delete-after                                 When synchronizing, delete files on destination after transferring (default)
-      --delete-before                                When synchronizing, delete files on destination before transferring
-      --delete-during                                When synchronizing, delete files during transfer
-      --delete-excluded                              Delete files on dest excluded from sync
-      --disable string                               Disable a comma separated list of features.  Use help to see a list.
-      --drive-acknowledge-abuse                      Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
-      --drive-allow-import-name-change               Allow the filetype to change when uploading Google docs (e.g. file.doc to file.docx). This will confuse sync and reupload every time.
-      --drive-alternate-export                       Use alternate export URLs for google documents export.,
-      --drive-auth-owner-only                        Only consider files owned by the authenticated user.
-      --drive-chunk-size SizeSuffix                  Upload chunk size. Must a power of 2 >= 256k. (default 8M)
-      --drive-client-id string                       Google Application Client Id
-      --drive-client-secret string                   Google Application Client Secret
-      --drive-export-formats string                  Comma separated list of preferred formats for downloading Google docs. (default "docx,xlsx,pptx,svg")
-      --drive-formats string                         Deprecated: see export_formats
-      --drive-impersonate string                     Impersonate this user when using a service account.
-      --drive-import-formats string                  Comma separated list of preferred formats for uploading Google docs.
-      --drive-keep-revision-forever                  Keep new head revision of each file forever.
-      --drive-list-chunk int                         Size of listing chunk 100-1000. 0 to disable. (default 1000)
-      --drive-pacer-burst int                        Number of API calls to allow without sleeping. (default 100)
-      --drive-pacer-min-sleep Duration               Minimum time to sleep between API calls. (default 100ms)
-      --drive-root-folder-id string                  ID of the root folder
-      --drive-scope string                           Scope that rclone should use when requesting access from drive.
-      --drive-service-account-credentials string     Service Account Credentials JSON blob
-      --drive-service-account-file string            Service Account Credentials JSON file path
-      --drive-shared-with-me                         Only show files that are shared with me.
-      --drive-skip-gdocs                             Skip google documents in all listings.
-      --drive-team-drive string                      ID of the Team Drive
-      --drive-trashed-only                           Only show files that are in the trash.
-      --drive-upload-cutoff SizeSuffix               Cutoff for switching to chunked upload (default 8M)
-      --drive-use-created-date                       Use file created date instead of modified date.,
-      --drive-use-trash                              Send files to the trash instead of deleting permanently. (default true)
-      --drive-v2-download-min-size SizeSuffix        If Object's are greater, use drive v2 API to download. (default off)
-      --dropbox-chunk-size SizeSuffix                Upload chunk size. (< 150M). (default 48M)
-      --dropbox-client-id string                     Dropbox App Client Id
-      --dropbox-client-secret string                 Dropbox App Client Secret
-      --dropbox-impersonate string                   Impersonate this user when using a business account.
-  -n, --dry-run                                      Do a trial run with no permanent changes
-      --dump DumpFlags                               List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles
-      --dump-bodies                                  Dump HTTP headers and bodies - may contain sensitive info
-      --dump-headers                                 Dump HTTP bodies - may contain sensitive info
-      --exclude stringArray                          Exclude files matching pattern
-      --exclude-from stringArray                     Read exclude patterns from file
-      --exclude-if-present string                    Exclude directories if filename is present
-      --fast-list                                    Use recursive list if available. Uses more memory but fewer transactions.
-      --files-from stringArray                       Read list of source-file names from file
-  -f, --filter stringArray                           Add a file-filtering rule
-      --filter-from stringArray                      Read filtering patterns from a file
-      --ftp-host string                              FTP host to connect to
-      --ftp-pass string                              FTP password
-      --ftp-port string                              FTP port, leave blank to use default (21)
-      --ftp-user string                              FTP username, leave blank for current username, $USER
-      --gcs-bucket-acl string                        Access Control List for new buckets.
-      --gcs-client-id string                         Google Application Client Id
-      --gcs-client-secret string                     Google Application Client Secret
-      --gcs-location string                          Location for the newly created buckets.
-      --gcs-object-acl string                        Access Control List for new objects.
-      --gcs-project-number string                    Project number.
-      --gcs-service-account-file string              Service Account Credentials JSON file path
-      --gcs-storage-class string                     The storage class to use when storing objects in Google Cloud Storage.
-  -h, --help                                         help for rclone
-      --http-url string                              URL of http host to connect to
-      --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container. (default 5G)
-      --hubic-client-id string                       Hubic Client Id
-      --hubic-client-secret string                   Hubic Client Secret
-      --hubic-no-chunk                               Don't chunk files during streaming upload.
-      --ignore-case                                  Ignore case in filters (case insensitive)
-      --ignore-checksum                              Skip post copy check of checksums.
-      --ignore-errors                                delete even if there are I/O errors
-      --ignore-existing                              Skip all files that exist on destination
-      --ignore-size                                  Ignore size when skipping use mod-time or checksum.
-  -I, --ignore-times                                 Don't skip files that match size and time - transfer all files
-      --immutable                                    Do not modify files. Fail if existing files have been modified.
-      --include stringArray                          Include files matching pattern
-      --include-from stringArray                     Read include patterns from file
-      --jottacloud-hard-delete                       Delete files permanently rather than putting them into the trash.
-      --jottacloud-md5-memory-limit SizeSuffix       Files bigger than this will be cached on disk to calculate the MD5 if required. (default 10M)
-      --jottacloud-mountpoint string                 The mountpoint to use.
-      --jottacloud-unlink                            Remove existing public link to file/folder with link command rather than creating.
-      --jottacloud-upload-resume-limit SizeSuffix    Files bigger than this can be resumed if the upload fail's. (default 10M)
-      --jottacloud-user string                       User Name:
-  -l, --links                                        Translate symlinks to/from regular files with a '.rclonelink' extension
-      --local-no-check-updated                       Don't check to see if the files change during upload
-      --local-no-unicode-normalization               Don't apply unicode normalization to paths and filenames (Deprecated)
-      --local-nounc string                           Disable UNC (long path names) conversion on Windows
-      --log-file string                              Log everything to this file
-      --log-format string                            Comma separated list of log format options (default "date,time")
-      --log-level string                             Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE")
-      --low-level-retries int                        Number of low level retries to do. (default 10)
-      --max-age Duration                             Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y (default off)
-      --max-backlog int                              Maximum number of objects in sync or check backlog. (default 10000)
-      --max-delete int                               When synchronizing, limit the number of deletes (default -1)
-      --max-depth int                                If set limits the recursion depth to this. (default -1)
-      --max-size SizeSuffix                          Only transfer files smaller than this in k or suffix b|k|M|G (default off)
-      --max-transfer SizeSuffix                      Maximum size of data to transfer. (default off)
-      --mega-debug                                   Output more debug from Mega.
-      --mega-hard-delete                             Delete files permanently rather than putting them into the trash.
-      --mega-pass string                             Password.
-      --mega-user string                             User name
-      --memprofile string                            Write memory profile to file
-      --min-age Duration                             Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
-      --min-size SizeSuffix                          Only transfer files bigger than this in k or suffix b|k|M|G (default off)
-      --modify-window duration                       Max time diff to be considered the same (default 1ns)
-      --no-check-certificate                         Do not verify the server SSL certificate. Insecure.
-      --no-gzip-encoding                             Don't set Accept-Encoding: gzip.
-      --no-traverse                                  Don't traverse destination file system on copy.
-      --no-update-modtime                            Don't update destination mod-time if files identical.
-  -x, --one-file-system                              Don't cross filesystem boundaries (unix/macOS only).
-      --onedrive-chunk-size SizeSuffix               Chunk size to upload files with - must be multiple of 320k. (default 10M)
-      --onedrive-client-id string                    Microsoft App Client Id
-      --onedrive-client-secret string                Microsoft App Client Secret
-      --onedrive-drive-id string                     The ID of the drive to use
-      --onedrive-drive-type string                   The type of the drive ( personal | business | documentLibrary )
-      --onedrive-expose-onenote-files                Set to make OneNote files show up in directory listings.
-      --opendrive-password string                    Password.
-      --opendrive-username string                    Username
-      --pcloud-client-id string                      Pcloud App Client Id
-      --pcloud-client-secret string                  Pcloud App Client Secret
-  -P, --progress                                     Show progress during transfer.
-      --qingstor-access-key-id string                QingStor Access Key ID
-      --qingstor-chunk-size SizeSuffix               Chunk size to use for uploading. (default 4M)
-      --qingstor-connection-retries int              Number of connection retries. (default 3)
-      --qingstor-endpoint string                     Enter a endpoint URL to connection QingStor API.
-      --qingstor-env-auth                            Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.
-      --qingstor-secret-access-key string            QingStor Secret Access Key (password)
-      --qingstor-upload-concurrency int              Concurrency for multipart uploads. (default 1)
-      --qingstor-upload-cutoff SizeSuffix            Cutoff for switching to chunked upload (default 200M)
-      --qingstor-zone string                         Zone to connect to.
-  -q, --quiet                                        Print as little stuff as possible
-      --rc                                           Enable the remote control server.
-      --rc-addr string                               IPaddress:Port or :Port to bind server to. (default "localhost:5572")
-      --rc-cert string                               SSL PEM key (concatenation of certificate and CA certificate)
-      --rc-client-ca string                          Client certificate authority to verify clients with
-      --rc-files string                              Path to local files to serve on the HTTP server.
-      --rc-htpasswd string                           htpasswd file - if not provided no authentication is done
-      --rc-key string                                SSL PEM Private key
-      --rc-max-header-bytes int                      Maximum size of request header (default 4096)
-      --rc-no-auth                                   Don't require auth for certain methods.
-      --rc-pass string                               Password for authentication.
-      --rc-realm string                              realm for authentication (default "rclone")
-      --rc-serve                                     Enable the serving of remote objects.
-      --rc-server-read-timeout duration              Timeout for server reading data (default 1h0m0s)
-      --rc-server-write-timeout duration             Timeout for server writing data (default 1h0m0s)
-      --rc-user string                               User name for authentication.
-      --retries int                                  Retry operations this many times if they fail (default 3)
-      --retries-sleep duration                       Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)
-      --s3-access-key-id string                      AWS Access Key ID.
-      --s3-acl string                                Canned ACL used when creating buckets and storing or copying objects.
-      --s3-bucket-acl string                         Canned ACL used when creating buckets.
-      --s3-chunk-size SizeSuffix                     Chunk size to use for uploading. (default 5M)
-      --s3-disable-checksum                          Don't store MD5 checksum with object metadata
-      --s3-endpoint string                           Endpoint for S3 API.
-      --s3-env-auth                                  Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
-      --s3-force-path-style                          If true use path style access if false use virtual hosted style. (default true)
-      --s3-location-constraint string                Location constraint - must be set to match the Region.
-      --s3-provider string                           Choose your S3 provider.
-      --s3-region string                             Region to connect to.
-      --s3-secret-access-key string                  AWS Secret Access Key (password)
-      --s3-server-side-encryption string             The server-side encryption algorithm used when storing this object in S3.
-      --s3-session-token string                      An AWS session token
-      --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key.
-      --s3-storage-class string                      The storage class to use when storing new objects in S3.
-      --s3-upload-concurrency int                    Concurrency for multipart uploads. (default 4)
-      --s3-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload (default 200M)
-      --s3-v2-auth                                   If true use v2 authentication.
-      --sftp-ask-password                            Allow asking for SFTP password when needed.
-      --sftp-disable-hashcheck                       Disable the execution of SSH commands to determine if remote file hashing is available.
-      --sftp-host string                             SSH host to connect to
-      --sftp-key-file string                         Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
-      --sftp-key-file-pass string                    The passphrase to decrypt the PEM-encoded private key file.
-      --sftp-key-use-agent                           When set forces the usage of the ssh-agent.
-      --sftp-pass string                             SSH password, leave blank to use ssh-agent.
-      --sftp-path-override string                    Override path used by SSH connection.
-      --sftp-port string                             SSH port, leave blank to use default (22)
-      --sftp-set-modtime                             Set the modified time on the remote if set. (default true)
-      --sftp-use-insecure-cipher                     Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.
-      --sftp-user string                             SSH username, leave blank for current username, ncw
-      --size-only                                    Skip based on size only, not mod-time or checksum
-      --skip-links                                   Don't warn about skipped symlinks.
-      --stats duration                               Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable) (default 1m0s)
-      --stats-file-name-length int                   Max file name length in stats. 0 for no limit (default 45)
-      --stats-log-level string                       Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO")
-      --stats-one-line                               Make the stats fit on one line.
-      --stats-unit string                            Show data rate in stats as either 'bits' or 'bytes'/s (default "bytes")
-      --streaming-upload-cutoff SizeSuffix           Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends. (default 100k)
-      --suffix string                                Suffix for use with --backup-dir.
-      --swift-application-credential-id string       Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)
-      --swift-application-credential-name string     Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)
-      --swift-application-credential-secret string   Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)
-      --swift-auth string                            Authentication URL for server (OS_AUTH_URL).
-      --swift-auth-token string                      Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)
-      --swift-auth-version int                       AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)
-      --swift-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container. (default 5G)
-      --swift-domain string                          User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)
-      --swift-endpoint-type string                   Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public")
-      --swift-env-auth                               Get swift credentials from environment variables in standard OpenStack form.
-      --swift-key string                             API key or password (OS_PASSWORD).
-      --swift-no-chunk                               Don't chunk files during streaming upload.
-      --swift-region string                          Region name - optional (OS_REGION_NAME)
-      --swift-storage-policy string                  The storage policy to use when creating a new container
-      --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
-      --swift-tenant string                          Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)
-      --swift-tenant-domain string                   Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)
-      --swift-tenant-id string                       Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)
-      --swift-user string                            User name to log in (OS_USERNAME).
-      --swift-user-id string                         User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).
-      --syslog                                       Use Syslog for logging
-      --syslog-facility string                       Facility for syslog, eg KERN,USER,... (default "DAEMON")
-      --timeout duration                             IO idle timeout (default 5m0s)
-      --tpslimit float                               Limit HTTP transactions per second to this.
-      --tpslimit-burst int                           Max burst of transactions for --tpslimit. (default 1)
-      --track-renames                                When synchronizing, track file renames and do a server side move if possible
-      --transfers int                                Number of file transfers to run in parallel. (default 4)
-      --union-remotes string                         List of space separated remotes.
-  -u, --update                                       Skip files that are newer on the destination.
-      --use-cookies                                  Enable session cookiejar.
-      --use-mmap                                     Use mmap allocator (see docs).
-      --use-server-modtime                           Use server modified time instead of object metadata
-      --user-agent string                            Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.46")
-  -v, --verbose count                                Print lots more stuff (repeat for more)
-  -V, --version                                      Print the version number
-      --webdav-bearer-token string                   Bearer token instead of user/pass (eg a Macaroon)
-      --webdav-pass string                           Password.
-      --webdav-url string                            URL of http host to connect to
-      --webdav-user string                           User name
-      --webdav-vendor string                         Name of the Webdav site/service/software you are using
-      --yandex-client-id string                      Yandex Client Id
-      --yandex-client-secret string                  Yandex Client Secret
-      --yandex-unlink                                Remove existing public link to file/folder with link command rather than creating.
+      --acd-auth-url string                        Auth server URL.
+      --acd-client-id string                       Amazon Application Client ID.
+      --acd-client-secret string                   Amazon Application Client Secret.
+      --acd-templink-threshold SizeSuffix          Files >= this size will be downloaded via their tempLink. (default 9G)
+      --acd-token-url string                       Token server url.
+      --acd-upload-wait-per-gb Duration            Additional time per GB to wait after a failed complete upload to see if it appears. (default 3m0s)
+      --alias-remote string                        Remote or path to alias.
+      --ask-password                               Allow prompt for password for encrypted configuration. (default true)
+      --auto-confirm                               If enabled, do not request console confirmation.
+      --azureblob-access-tier string               Access tier of blob: hot, cool or archive.
+      --azureblob-account string                   Storage Account Name (leave blank to use connection string or SAS URL)
+      --azureblob-chunk-size SizeSuffix            Upload chunk size (<= 100MB). (default 4M)
+      --azureblob-endpoint string                  Endpoint for the service
+      --azureblob-key string                       Storage Account Key (leave blank to use connection string or SAS URL)
+      --azureblob-list-chunk int                   Size of blob list. (default 5000)
+      --azureblob-sas-url string                   SAS URL for container level access only
+      --azureblob-upload-cutoff SizeSuffix         Cutoff for switching to chunked upload (<= 256MB). (default 256M)
+      --b2-account string                          Account ID or Application Key ID
+      --b2-chunk-size SizeSuffix                   Upload chunk size. Must fit in memory. (default 96M)
+      --b2-endpoint string                         Endpoint for the service.
+      --b2-hard-delete                             Permanently delete files on remote removal, otherwise hide files.
+      --b2-key string                              Application Key
+      --b2-test-mode string                        A flag string for X-Bz-Test-Mode header for debugging.
+      --b2-upload-cutoff SizeSuffix                Cutoff for switching to chunked upload. (default 200M)
+      --b2-versions                                Include old versions in directory listings.
+      --backup-dir string                          Make backups into hierarchy based in DIR.
+      --bind string                                Local address to bind to for outgoing connections, IPv4, IPv6 or name.
+      --box-client-id string                       Box App Client Id.
+      --box-client-secret string                   Box App Client Secret
+      --box-commit-retries int                     Max number of times to try committing a multipart file. (default 100)
+      --box-upload-cutoff SizeSuffix               Cutoff for switching to multipart upload (>= 50MB). (default 50M)
+      --buffer-size int                            In memory buffer size when reading files for each --transfer. (default 16M)
+      --bwlimit BwTimetable                        Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.
+      --cache-chunk-clean-interval Duration        How often should the cache perform cleanups of the chunk storage. (default 1m0s)
+      --cache-chunk-no-memory                      Disable the in-memory cache for storing chunks during streaming.
+      --cache-chunk-path string                    Directory to cache chunk files. (default "$HOME/.cache/rclone/cache-backend")
+      --cache-chunk-size SizeSuffix                The size of a chunk (partial file data). (default 5M)
+      --cache-chunk-total-size SizeSuffix          The total size that the chunks can take up on the local disk. (default 10G)
+      --cache-db-path string                       Directory to store file structure metadata DB. (default "$HOME/.cache/rclone/cache-backend")
+      --cache-db-purge                             Clear all the cached data for this remote on start.
+      --cache-db-wait-time Duration                How long to wait for the DB to be available - 0 is unlimited (default 1s)
+      --cache-dir string                           Directory rclone will use for caching. (default "$HOME/.cache/rclone")
+      --cache-info-age Duration                    How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
+      --cache-plex-insecure string                 Skip all certificate verifications when connecting to the Plex server
+      --cache-plex-password string                 The password of the Plex user
+      --cache-plex-url string                      The URL of the Plex server
+      --cache-plex-username string                 The username of the Plex user
+      --cache-read-retries int                     How many times to retry a read from a cache storage. (default 10)
+      --cache-remote string                        Remote to cache.
+      --cache-rps int                              Limits the number of requests per second to the source FS (-1 to disable) (default -1)
+      --cache-tmp-upload-path string               Directory to keep temporary files until they are uploaded.
+      --cache-tmp-wait-time Duration               How long should files be stored in local cache before being uploaded (default 15s)
+      --cache-workers int                          How many workers should run in parallel to download chunks. (default 4)
+      --cache-writes                               Cache file data on writes through the FS
+      --checkers int                               Number of checkers to run in parallel. (default 8)
+  -c, --checksum                                   Skip based on checksum & size, not mod-time & size
+      --config string                              Config file. (default "/home/ncw/.rclone.conf")
+      --contimeout duration                        Connect timeout (default 1m0s)
+  -L, --copy-links                                 Follow symlinks and copy the pointed to item.
+      --cpuprofile string                          Write cpu profile to file
+      --crypt-directory-name-encryption            Option to either encrypt directory names or leave them intact. (default true)
+      --crypt-filename-encryption string           How to encrypt the filenames. (default "standard")
+      --crypt-password string                      Password or pass phrase for encryption.
+      --crypt-password2 string                     Password or pass phrase for salt. Optional but recommended.
+      --crypt-remote string                        Remote to encrypt/decrypt.
+      --crypt-show-mapping                         For all files listed show how the names encrypt.
+      --delete-after                               When synchronizing, delete files on destination after transferring (default)
+      --delete-before                              When synchronizing, delete files on destination before transferring
+      --delete-during                              When synchronizing, delete files during transfer
+      --delete-excluded                            Delete files on dest excluded from sync
+      --disable string                             Disable a comma separated list of features.  Use help to see a list.
+      --drive-acknowledge-abuse                    Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
+      --drive-allow-import-name-change             Allow the filetype to change when uploading Google docs (e.g. file.doc to file.docx). This will confuse sync and reupload every time.
+      --drive-alternate-export                     Use alternate export URLs for google documents export.,
+      --drive-auth-owner-only                      Only consider files owned by the authenticated user.
+      --drive-chunk-size SizeSuffix                Upload chunk size. Must a power of 2 >= 256k. (default 8M)
+      --drive-client-id string                     Google Application Client Id
+      --drive-client-secret string                 Google Application Client Secret
+      --drive-export-formats string                Comma separated list of preferred formats for downloading Google docs. (default "docx,xlsx,pptx,svg")
+      --drive-formats string                       Deprecated: see export_formats
+      --drive-impersonate string                   Impersonate this user when using a service account.
+      --drive-import-formats string                Comma separated list of preferred formats for uploading Google docs.
+      --drive-keep-revision-forever                Keep new head revision of each file forever.
+      --drive-list-chunk int                       Size of listing chunk 100-1000. 0 to disable. (default 1000)
+      --drive-root-folder-id string                ID of the root folder
+      --drive-scope string                         Scope that rclone should use when requesting access from drive.
+      --drive-service-account-credentials string   Service Account Credentials JSON blob
+      --drive-service-account-file string          Service Account Credentials JSON file path
+      --drive-shared-with-me                       Only show files that are shared with me.
+      --drive-skip-gdocs                           Skip google documents in all listings.
+      --drive-team-drive string                    ID of the Team Drive
+      --drive-trashed-only                         Only show files that are in the trash.
+      --drive-upload-cutoff SizeSuffix             Cutoff for switching to chunked upload (default 8M)
+      --drive-use-created-date                     Use file created date instead of modified date.,
+      --drive-use-trash                            Send files to the trash instead of deleting permanently. (default true)
+      --drive-v2-download-min-size SizeSuffix      If Object's are greater, use drive v2 API to download. (default off)
+      --dropbox-chunk-size SizeSuffix              Upload chunk size. (< 150M). (default 48M)
+      --dropbox-client-id string                   Dropbox App Client Id
+      --dropbox-client-secret string               Dropbox App Client Secret
+      --dropbox-impersonate string                 Impersonate this user when using a business account.
+  -n, --dry-run                                    Do a trial run with no permanent changes
+      --dump string                                List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles
+      --dump-bodies                                Dump HTTP headers and bodies - may contain sensitive info
+      --dump-headers                               Dump HTTP bodies - may contain sensitive info
+      --exclude stringArray                        Exclude files matching pattern
+      --exclude-from stringArray                   Read exclude patterns from file
+      --exclude-if-present string                  Exclude directories if filename is present
+      --fast-list                                  Use recursive list if available. Uses more memory but fewer transactions.
+      --files-from stringArray                     Read list of source-file names from file
+  -f, --filter stringArray                         Add a file-filtering rule
+      --filter-from stringArray                    Read filtering patterns from a file
+      --ftp-host string                            FTP host to connect to
+      --ftp-pass string                            FTP password
+      --ftp-port string                            FTP port, leave blank to use default (21)
+      --ftp-user string                            FTP username, leave blank for current username, $USER
+      --gcs-bucket-acl string                      Access Control List for new buckets.
+      --gcs-client-id string                       Google Application Client Id
+      --gcs-client-secret string                   Google Application Client Secret
+      --gcs-location string                        Location for the newly created buckets.
+      --gcs-object-acl string                      Access Control List for new objects.
+      --gcs-project-number string                  Project number.
+      --gcs-service-account-file string            Service Account Credentials JSON file path
+      --gcs-storage-class string                   The storage class to use when storing objects in Google Cloud Storage.
+  -h, --help                                       help for rclone
+      --http-url string                            URL of http host to connect to
+      --hubic-chunk-size SizeSuffix                Above this size files will be chunked into a _segments container. (default 5G)
+      --hubic-client-id string                     Hubic Client Id
+      --hubic-client-secret string                 Hubic Client Secret
+      --ignore-case                                Ignore case in filters (case insensitive)
+      --ignore-checksum                            Skip post copy check of checksums.
+      --ignore-errors                              delete even if there are I/O errors
+      --ignore-existing                            Skip all files that exist on destination
+      --ignore-size                                Ignore size when skipping use mod-time or checksum.
+  -I, --ignore-times                               Don't skip files that match size and time - transfer all files
+      --immutable                                  Do not modify files. Fail if existing files have been modified.
+      --include stringArray                        Include files matching pattern
+      --include-from stringArray                   Read include patterns from file
+      --jottacloud-hard-delete                     Delete files permanently rather than putting them into the trash.
+      --jottacloud-md5-memory-limit SizeSuffix     Files bigger than this will be cached on disk to calculate the MD5 if required. (default 10M)
+      --jottacloud-mountpoint string               The mountpoint to use.
+      --jottacloud-pass string                     Password.
+      --jottacloud-unlink                          Remove existing public link to file/folder with link command rather than creating.
+      --jottacloud-user string                     User Name
+      --local-no-check-updated                     Don't check to see if the files change during upload
+      --local-no-unicode-normalization             Don't apply unicode normalization to paths and filenames (Deprecated)
+      --local-nounc string                         Disable UNC (long path names) conversion on Windows
+      --log-file string                            Log everything to this file
+      --log-format string                          Comma separated list of log format options (default "date,time")
+      --log-level string                           Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE")
+      --low-level-retries int                      Number of low level retries to do. (default 10)
+      --max-age duration                           Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y (default off)
+      --max-backlog int                            Maximum number of objects in sync or check backlog. (default 10000)
+      --max-delete int                             When synchronizing, limit the number of deletes (default -1)
+      --max-depth int                              If set limits the recursion depth to this. (default -1)
+      --max-size int                               Only transfer files smaller than this in k or suffix b|k|M|G (default off)
+      --max-transfer int                           Maximum size of data to transfer. (default off)
+      --mega-debug                                 Output more debug from Mega.
+      --mega-hard-delete                           Delete files permanently rather than putting them into the trash.
+      --mega-pass string                           Password.
+      --mega-user string                           User name
+      --memprofile string                          Write memory profile to file
+      --min-age duration                           Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
+      --min-size int                               Only transfer files bigger than this in k or suffix b|k|M|G (default off)
+      --modify-window duration                     Max time diff to be considered the same (default 1ns)
+      --no-check-certificate                       Do not verify the server SSL certificate. Insecure.
+      --no-gzip-encoding                           Don't set Accept-Encoding: gzip.
+      --no-traverse                                Obsolete - does nothing.
+      --no-update-modtime                          Don't update destination mod-time if files identical.
+  -x, --one-file-system                            Don't cross filesystem boundaries (unix/macOS only).
+      --onedrive-chunk-size SizeSuffix             Chunk size to upload files with - must be multiple of 320k. (default 10M)
+      --onedrive-client-id string                  Microsoft App Client Id
+      --onedrive-client-secret string              Microsoft App Client Secret
+      --onedrive-drive-id string                   The ID of the drive to use
+      --onedrive-drive-type string                 The type of the drive ( personal | business | documentLibrary )
+      --onedrive-expose-onenote-files              Set to make OneNote files show up in directory listings.
+      --opendrive-password string                  Password.
+      --opendrive-username string                  Username
+      --pcloud-client-id string                    Pcloud App Client Id
+      --pcloud-client-secret string                Pcloud App Client Secret
+  -P, --progress                                   Show progress during transfer.
+      --qingstor-access-key-id string              QingStor Access Key ID
+      --qingstor-connection-retries int            Number of connection retries. (default 3)
+      --qingstor-endpoint string                   Enter a endpoint URL to connection QingStor API.
+      --qingstor-env-auth                          Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.
+      --qingstor-secret-access-key string          QingStor Secret Access Key (password)
+      --qingstor-zone string                       Zone to connect to.
+  -q, --quiet                                      Print as little stuff as possible
+      --rc                                         Enable the remote control server.
+      --rc-addr string                             IPaddress:Port or :Port to bind server to. (default "localhost:5572")
+      --rc-cert string                             SSL PEM key (concatenation of certificate and CA certificate)
+      --rc-client-ca string                        Client certificate authority to verify clients with
+      --rc-files string                            Path to local files to serve on the HTTP server.
+      --rc-htpasswd string                         htpasswd file - if not provided no authentication is done
+      --rc-key string                              SSL PEM Private key
+      --rc-max-header-bytes int                    Maximum size of request header (default 4096)
+      --rc-no-auth                                 Don't require auth for certain methods.
+      --rc-pass string                             Password for authentication.
+      --rc-realm string                            realm for authentication (default "rclone")
+      --rc-serve                                   Enable the serving of remote objects.
+      --rc-server-read-timeout duration            Timeout for server reading data (default 1h0m0s)
+      --rc-server-write-timeout duration           Timeout for server writing data (default 1h0m0s)
+      --rc-user string                             User name for authentication.
+      --retries int                                Retry operations this many times if they fail (default 3)
+      --retries-sleep duration                     Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)
+      --s3-access-key-id string                    AWS Access Key ID.
+      --s3-acl string                              Canned ACL used when creating buckets and storing or copying objects.
+      --s3-chunk-size SizeSuffix                   Chunk size to use for uploading. (default 5M)
+      --s3-disable-checksum                        Don't store MD5 checksum with object metadata
+      --s3-endpoint string                         Endpoint for S3 API.
+      --s3-env-auth                                Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+      --s3-force-path-style                        If true use path style access if false use virtual hosted style. (default true)
+      --s3-location-constraint string              Location constraint - must be set to match the Region.
+      --s3-provider string                         Choose your S3 provider.
+      --s3-region string                           Region to connect to.
+      --s3-secret-access-key string                AWS Secret Access Key (password)
+      --s3-server-side-encryption string           The server-side encryption algorithm used when storing this object in S3.
+      --s3-session-token string                    An AWS session token
+      --s3-sse-kms-key-id string                   If using KMS ID you must provide the ARN of Key.
+      --s3-storage-class string                    The storage class to use when storing new objects in S3.
+      --s3-upload-concurrency int                  Concurrency for multipart uploads. (default 2)
+      --s3-v2-auth                                 If true use v2 authentication.
+      --sftp-ask-password                          Allow asking for SFTP password when needed.
+      --sftp-disable-hashcheck                     Disable the execution of SSH commands to determine if remote file hashing is available.
+      --sftp-host string                           SSH host to connect to
+      --sftp-key-file string                       Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.
+      --sftp-pass string                           SSH password, leave blank to use ssh-agent.
+      --sftp-path-override string                  Override path used by SSH connection.
+      --sftp-port string                           SSH port, leave blank to use default (22)
+      --sftp-set-modtime                           Set the modified time on the remote if set. (default true)
+      --sftp-use-insecure-cipher                   Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.
+      --sftp-user string                           SSH username, leave blank for current username, ncw
+      --size-only                                  Skip based on size only, not mod-time or checksum
+      --skip-links                                 Don't warn about skipped symlinks.
+      --stats duration                             Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable) (default 1m0s)
+      --stats-file-name-length int                 Max file name length in stats. 0 for no limit (default 40)
+      --stats-log-level string                     Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO")
+      --stats-one-line                             Make the stats fit on one line.
+      --stats-unit string                          Show data rate in stats as either 'bits' or 'bytes'/s (default "bytes")
+      --streaming-upload-cutoff int                Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends. (default 100k)
+      --suffix string                              Suffix for use with --backup-dir.
+      --swift-auth string                          Authentication URL for server (OS_AUTH_URL).
+      --swift-auth-token string                    Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)
+      --swift-auth-version int                     AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)
+      --swift-chunk-size SizeSuffix                Above this size files will be chunked into a _segments container. (default 5G)
+      --swift-domain string                        User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)
+      --swift-endpoint-type string                 Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public")
+      --swift-env-auth                             Get swift credentials from environment variables in standard OpenStack form.
+      --swift-key string                           API key or password (OS_PASSWORD).
+      --swift-region string                        Region name - optional (OS_REGION_NAME)
+      --swift-storage-policy string                The storage policy to use when creating a new container
+      --swift-storage-url string                   Storage URL - optional (OS_STORAGE_URL)
+      --swift-tenant string                        Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)
+      --swift-tenant-domain string                 Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)
+      --swift-tenant-id string                     Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)
+      --swift-user string                          User name to log in (OS_USERNAME).
+      --swift-user-id string                       User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).
+      --syslog                                     Use Syslog for logging
+      --syslog-facility string                     Facility for syslog, eg KERN,USER,... (default "DAEMON")
+      --timeout duration                           IO idle timeout (default 5m0s)
+      --tpslimit float                             Limit HTTP transactions per second to this.
+      --tpslimit-burst int                         Max burst of transactions for --tpslimit. (default 1)
+      --track-renames                              When synchronizing, track file renames and do a server side move if possible
+      --transfers int                              Number of file transfers to run in parallel. (default 4)
+      --union-remotes string                       List of space separated remotes.
+  -u, --update                                     Skip files that are newer on the destination.
+      --use-server-modtime                         Use server modified time instead of object metadata
+      --user-agent string                          Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.45")
+  -v, --verbose count                              Print lots more stuff (repeat for more)
+  -V, --version                                    Print the version number
+      --webdav-bearer-token string                 Bearer token instead of user/pass (eg a Macaroon)
+      --webdav-pass string                         Password.
+      --webdav-url string                          URL of http host to connect to
+      --webdav-user string                         User name
+      --webdav-vendor string                       Name of the Webdav site/service/software you are using
+      --yandex-client-id string                    Yandex Client Id
+      --yandex-client-secret string                Yandex Client Secret
+      --yandex-unlink                              Remove existing public link to file/folder with link command rather than creating.
 ```
 
 ### SEE ALSO
@@ -373,4 +355,4 @@ rclone [flags]
 * [rclone tree](/commands/rclone_tree/)	 - List the contents of the remote in a tree like fashion.
 * [rclone version](/commands/rclone_version/)	 - Show the version number.
 
-###### Auto generated by spf13/cobra on 9-Feb-2019
+###### Auto generated by spf13/cobra on 24-Nov-2018