mount,cmount: run Release asynchronously

Before this change the mount and cmount would run Release
synchronously.  This would mean that it would wait for files to be
closed (eg uploaded) before returning to the kernel.

However Release is already running asynchronously from userspace so
this commit changes it to do the functionality of Release
asynchronously too.

This should fix libfuse blocking when Release is active and it is
asked to do something else with a file.

Forum: https://forum.rclone.org/t/vfs-cache-mode-writes-upload-expected-behaviour/8014

.golangci.yml

@@ -0,0 +1,26 @@
+# golangci-lint configuration options
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goimports
+    - golint
+    - ineffassign
+    - structcheck
+    - varcheck
+    - govet
+    - unconvert
+    #- prealloc
+    #- maligned
+  disable-all: true
+
+issues:
+  # Enable some lints excluded by default
+  exclude-use-default: false
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0

.gometalinter.json

@@ -1,14 +0,0 @@
-{
-    "Enable": [
-        "deadcode",
-        "errcheck",
-        "goimports",
-        "golint",
-        "ineffassign",
-        "structcheck",
-        "varcheck",
-        "vet"
-    ],
-    "EnableGC": true,
-    "Vendor": true
-}

.travis.yml

@@ -1,52 +1,100 @@
+---
 language: go
 sudo: required
-dist: trusty
+dist: xenial
 os:
-- linux
-go:
-- 1.8.x
-- 1.9.x
-- 1.10.x
-- 1.11.x
-- tip
+  - linux
 go_import_path: github.com/ncw/rclone
 before_install:
-- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
-- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
+  - git fetch --unshallow --tags
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      sudo modprobe fuse
+      sudo chmod 666 /dev/fuse
+      sudo chown root:$USER /etc/fuse.conf
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+      brew update
+      brew tap caskroom/cask
+      brew cask install osxfuse
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
+      choco install -y winfsp zip make
+      cd ../..                  # fix crlf in git checkout
+      mv $TRAVIS_REPO_SLUG _old
+      git config --global core.autocrlf false
+      git clone _old $TRAVIS_REPO_SLUG
+      cd $TRAVIS_REPO_SLUG
+    fi
 install:
-- git fetch --unshallow --tags
-- make vars
-- make build_dep
-script:
-- make check
-- make quicktest
-- make compile_all
+  - make vars
 env:
   global:
     - GOTAGS=cmount
+    - GO111MODULE=off
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
     - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-    - fuse
-    - libfuse-dev
-    - rpm
-    - pkg-config
+      - fuse
+      - libfuse-dev
+      - rpm
+      - pkg-config
 cache:
   directories:
     - $HOME/.cache/go-build
 matrix:
   allow_failures:
-  - go: tip
+    - go: tip
   include:
-  - os: osx
-    go: 1.11.x
-    env: GOTAGS=""
-    cache:
-      directories:
-        - $HOME/Library/Caches/go-build
+    - go: 1.9.x
+      script:
+        - make quicktest
+    - go: 1.10.x
+      script:
+        - make quicktest
+    - go: 1.11.x
+      script:
+        - make quicktest
+    - go: 1.12.x
+      env:
+        - GOTAGS=cmount
+      script:
+        - make build_dep
+        - make check
+        - make quicktest
+        - make racequicktest
+        - make compile_all
+    - os: osx
+      go: 1.12.x
+      env:
+        - GOTAGS=  # cmount doesn't work on osx travis for some reason
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+      script:
+        - make
+        - make quicktest
+        - make racequicktest
+    # - os: windows
+    #   go: 1.12.x
+    #   env:
+    #     - GOTAGS=cmount
+    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
+    #   #filter_secrets: false     # works around a problem with secrets under windows
+    #   cache:
+    #     directories:
+    #       - ${LocalAppData}/go-build
+    #   script:
+    #     - make
+    #     - make quicktest
+    #     - make racequicktest
+    - go: tip
+      script:
+        - make quicktest
+
 deploy:
   provider: script
   script: make travis_beta
@@ -54,5 +102,5 @@ deploy:
   on:
     repo: ncw/rclone
     all_branches: true
-    go: 1.11.x
-    condition: $TRAVIS_PULL_REQUEST == false
+    go: 1.12.x
+    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"

CONTRIBUTING.md

@@ -135,7 +135,7 @@ then change into the project root and run
 
     make test
 
-This command is run daily on the the integration test server. You can
+This command is run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##

Makefile

@@ -11,14 +11,12 @@ ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	BRANCH_PATH :=
 endif
 TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
 ifneq ($(TAG),$(LAST_TAG))
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
-# Run full tests if go >= go1.11
-FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -26,6 +24,7 @@ BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
+LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
 .PHONY: rclone vars version
@@ -42,7 +41,6 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
-	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -57,38 +55,19 @@ test:	rclone
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-ifdef FULL_TESTS
+
+racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
-endif
 
 # Do source code quality checks
 check:	rclone
-ifdef FULL_TESTS
-	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
-	errcheck $(BUILDTAGS) ./...
-	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
-	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl|ApplicationCredentialId)' ; test $$? -eq 1
-else
-	@echo Skipping source quality tests as version of go too old
-endif
-
-gometalinter_install:
-	go get -u github.com/alecthomas/gometalinter
-	gometalinter --install --update
-
-# We aren't using gometalinter as the default linter yet because
-# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
-# 2. can't get -printfuncs working with the vet linter
-gometalinter:
-	gometalinter ./...
+	@echo "-- START CODE QUALITY REPORT -------------------------------"
+	@golangci-lint run $(LINTTAGS) ./...
+	@echo "-- END CODE QUALITY REPORT ---------------------------------"
 
 # Get the build dependencies
 build_dep:
-ifdef FULL_TESTS
-	go get -u github.com/kisielk/errcheck
-	go get -u golang.org/x/tools/cmd/goimports
-	go get -u golang.org/x/lint/golint
-endif
+	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
 # Get the release dependencies
 release_dep:
@@ -172,11 +151,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
-else
-	@echo Skipping compile all as version of go too old
-endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -196,10 +171,15 @@ BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
+ifeq ($(TRAVIS_OS_NAME),windows)
+# BUILD_FLAGS := -include "^windows/" -cgo
+# 386 doesn't build yet
+	BUILD_FLAGS := -include "^windows/amd64" -cgo
+endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
@@ -211,7 +191,7 @@ endif
 
 # Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync $(BETA_UPLOAD) build/
+	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server -v -w

README.md

@@ -7,11 +7,11 @@
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/) | 
-[G+](https://google.com/+RcloneOrg)
 
 [![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
 [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
 [![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/ncw/rclone)](https://goreportcard.com/report/github.com/ncw/rclone)
 [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
 # Rclone
@@ -36,6 +36,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
@@ -44,7 +45,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
+  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
@@ -62,13 +63,13 @@ Please see [the full list of all storage providers and their features](https://r
 
 ## Features
 
-  * MD5/SHA1 hashes checked at all times for file integrity
+  * MD5/SHA-1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, eg two different cloud accounts
+  * Can sync to and from network, e.g. two different cloud accounts
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))

RELEASE.md

@@ -11,7 +11,7 @@ Making a release
   * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX"
+  * git commit -a -v -m "Version v1.XX.0"
   * make retag
   * git push --tags origin master
   * # Wait for the appveyor and travis builds to complete then...
@@ -27,6 +27,7 @@ Making a release
 
 Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
+      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files

backend/alias/alias.go

@@ -30,7 +30,7 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs contstructs an Fs from the path.
+// NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {

backend/alias/alias_internal_test.go

@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
+			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}

backend/all/all.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/jottacloud"
+	_ "github.com/ncw/rclone/backend/koofr"
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -155,7 +154,7 @@ type Fs struct {
 	noAuthClient *http.Client       // unauthenticated http client
 	root         string             // the path we are working on
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	trueRootID   string             // ID of true root directory
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
@@ -273,7 +272,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:         root,
 		opt:          *opt,
 		c:            c,
-		pacer:        pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
+		pacer:        fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
 		noAuthClient: fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1093,7 +1092,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		if !bigObject {
 			in, resp, err = file.OpenHeaders(headers)
 		} else {
-			in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers)
+			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
 		return o.fs.shouldRetry(resp, err)
 	})

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 
@@ -77,7 +77,7 @@ func init() {
 		}, {
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
-			Default:  fs.SizeSuffix(defaultUploadCutoff),
+			Default:  defaultUploadCutoff,
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -85,7 +85,7 @@ func init() {
 
 Note that this is stored in memory and there may be up to
 "--transfers" chunks stored at once in memory.`,
-			Default:  fs.SizeSuffix(defaultChunkSize),
+			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
 			Name: "list_chunk",
@@ -144,7 +144,7 @@ type Fs struct {
 	containerOKMu    sync.Mutex            // mutex to protect container OK
 	containerOK      bool                  // true if we have created the container
 	containerDeleted bool                  // true if we have deleted the container
-	pacer            *pacer.Pacer          // To pace and retry the API calls
+	pacer            *fs.Pacer             // To pace and retry the API calls
 	uploadToken      *pacer.TokenDispenser // control concurrency
 }
 
@@ -307,7 +307,7 @@ func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline
 	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -347,7 +347,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:         *opt,
 		container:   container,
 		root:        directory,
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetPacer(pacer.S3Pacer),
+		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 		client:      fshttp.NewClient(fs.Config),
 	}
@@ -392,7 +392,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
 			}
 
-			container = parts.ContainerName
+			f.container = parts.ContainerName
 			containerURL = azblob.NewContainerURL(*u, pipeline)
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -1038,7 +1038,7 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetProper
 	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
 	o.mimeType = info.ContentType()
 	o.size = size
-	o.modTime = time.Time(info.LastModified())
+	o.modTime = info.LastModified()
 	o.accessTier = azblob.AccessTierType(info.AccessTier())
 	o.setMetadata(metadata)
 
@@ -1104,12 +1104,6 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
-// timeString returns modTime as the number of milliseconds
-// elapsed since January 1, 1970 UTC as a decimal string.
-func timeString(modTime time.Time) string {
-	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
-}
-
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1392,16 +1386,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	blob := o.getBlobReference()
 	httpHeaders := azblob.BlobHTTPHeaders{}
 	httpHeaders.ContentType = fs.MimeType(o)
-	// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
-	// MD5 only for PutBlob requests
-	if size < int64(o.fs.opt.UploadCutoff) {
-		if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
-			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-			if err == nil {
-				httpHeaders.ContentMD5 = sourceMD5bytes
-			} else {
-				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
-			}
+	// Compute the Content-MD5 of the file, for multiparts uploads it
+	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
+	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
+	// 		 in order to validate its integrity during transport
+	if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
+		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+		if err == nil {
+			httpHeaders.ContentMD5 = sourceMD5bytes
+		} else {
+			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
 		}
 	}
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris !go1.8
+// +build plan9 solaris
 
 package azureblob

backend/b2/api/types.go

@@ -17,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal statisfies the Fatal interface
+// Fatal satisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
 	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is unitialised
+// IsZero returns true if the timestamp is uninitialized
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }

backend/b2/b2.go

@@ -108,7 +108,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 Files above this size will be uploaded in chunks of "--b2-chunk-size".
 
 This value should be set no larger than 4.657GiB (== 5GB).`,
-			Default:  fs.SizeSuffix(defaultUploadCutoff),
+			Default:  defaultUploadCutoff,
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -117,14 +117,22 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
 When uploading large files, chunk the file into this size.  Note that
 these chunks are buffered in memory and there might a maximum of
 "--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimim size.`,
-			Default:  fs.SizeSuffix(defaultChunkSize),
+minimum size.`,
+			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
 			Name:     "disable_checksum",
 			Help:     `Disable checksums for large (> upload cutoff) files`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "download_url",
+			Help: `Custom endpoint for downloads.
+
+This is usually set to a Cloudflare CDN URL as Backblaze offers
+free egress for data downloaded through the Cloudflare network.
+Leave blank if you want to use the endpoint provided by Backblaze.`,
+			Advanced: true,
 		}},
 	})
 }
@@ -140,6 +148,7 @@ type Options struct {
 	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
 	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
 	DisableCheckSum bool          `config:"disable_checksum"`
+	DownloadURL     string        `config:"download_url"`
 }
 
 // Fs represents a remote b2 server
@@ -158,7 +167,7 @@ type Fs struct {
 	uploadMu      sync.Mutex                   // lock for upload variable
 	uploads       []*api.GetUploadURLResponse  // result of get upload URL calls
 	authMu        sync.Mutex                   // lock for authorizing the account
-	pacer         *pacer.Pacer                 // To pace and retry the API calls
+	pacer         *fs.Pacer                    // To pace and retry the API calls
 	bufferTokens  chan []byte                  // control concurrency of multipart uploads
 }
 
@@ -242,13 +251,7 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 				fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err)
 			}
 		}
-		retryAfterDuration := time.Duration(retryAfter) * time.Second
-		if f.pacer.GetSleep() < retryAfterDuration {
-			fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err)
-			// We set 1/2 the value here because the pacer will double it immediately
-			f.pacer.SetSleep(retryAfterDuration / 2)
-		}
-		return true, err
+		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Second)
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -319,7 +322,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -354,7 +357,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		srv:    rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -949,6 +952,13 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
+		if apiErr, ok := err.(*api.Error); ok {
+			if apiErr.Code == "already_hidden" {
+				// sometimes eventual consistency causes this, so
+				// ignore this error since it is harmless
+				return nil
+			}
+		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1203,7 +1213,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return err
+		return nil
 	}
 	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
 	return nil
@@ -1296,9 +1306,17 @@ var _ io.ReadCloser = &openFile{}
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	opts := rest.Opts{
 		Method:  "GET",
-		RootURL: o.fs.info.DownloadURL,
 		Options: options,
 	}
+
+	// Use downloadUrl from backblaze if downloadUrl is not set
+	// otherwise use the custom downloadUrl
+	if o.fs.opt.DownloadURL == "" {
+		opts.RootURL = o.fs.info.DownloadURL
+	} else {
+		opts.RootURL = o.fs.opt.DownloadURL
+	}
+
 	// Download by id if set otherwise by name
 	if o.id != "" {
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
@@ -1459,7 +1477,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// Content-Type b2/x-auto to automatically set the stored Content-Type
 	// post upload. In the case where a file extension is absent or the
 	// lookup fails, the Content-Type is set to application/octet-stream. The
-	// Content-Type mappings can be purused here.
+	// Content-Type mappings can be pursued here.
 	//
 	// X-Bz-Content-Sha1
 	// required

backend/box/api/types.go

@@ -45,7 +45,7 @@ type Error struct {
 	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,7 +57,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo

backend/box/box.go

@@ -111,7 +111,7 @@ type Fs struct {
 	features     *fs.Features          // optional features
 	srv          *rest.Client          // the connection to the one drive server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *pacer.Pacer          // pacer for API calls
+	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
 }
@@ -171,13 +171,13 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRety := false
+	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRety = true
+		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // substitute reserved characters for box
@@ -260,7 +260,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:        root,
 		opt:         *opt,
 		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{
@@ -530,10 +530,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return exisitingObj, exisitingObj.Update(in, src, options...)
+		return existingObj, existingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)

backend/box/upload.go

@@ -211,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= int64(chunkSize) {
-			reqSize = int64(chunkSize)
+		if reqSize >= chunkSize {
+			reqSize = chunkSize
 		}
 
 		// Make a block of memory

backend/cache/cache.go

@@ -576,7 +576,7 @@ The slice indices are similar to Python slices: start[:end]
 
 start is the 0 based chunk number from the beginning of the file
 to fetch inclusive. end is 0 based chunk number from the beginning
-of the file to fetch exclisive.
+of the file to fetch exclusive.
 Both values can be negative, in which case they count from the back
 of the file. The value "-5:" represents the last 5 chunks of a file.
 
@@ -870,7 +870,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 	}
 }
 
-// ChangeNotify can subsribe multiple callers
+// ChangeNotify can subscribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
@@ -1191,7 +1191,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1287,7 +1287,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1549,7 +1549,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 
 	if srcObj.isTempFile() {
-		// we check if the feature is stil active
+		// we check if the feature is still active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantCopy
@@ -1625,7 +1625,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 
 	// if this is a temp object then we perform the changes locally
 	if srcObj.isTempFile() {
-		// we check if the feature is stil active
+		// we check if the feature is still active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantMove

backend/cache/cache_internal_test.go

@@ -387,10 +387,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), int64(testSize))
+	require.Equal(t, o.Size(), testSize)
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -726,6 +726,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// Call the rc function
 	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
+	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -735,18 +736,21 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
-	li1, err := runInstance.list(t, rootFs, "")
+	_, err = runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 
 	// create some rand test data
 	testData2 := randStringBytes(int(chunkSize))
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
 
 	// list should have 1 item only
-	li1, err = runInstance.list(t, rootFs, "")
+	li1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 	require.Len(t, li1, 1)
 
 	// Call the rc function
 	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
+	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -754,6 +758,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// list should have 2 items now
 	li2, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 	require.Len(t, li2, 2)
 }
 
@@ -1490,7 +1495,8 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 	var err error
 
 	if r.useMount {
-		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		var f *os.File
+		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
@@ -1500,7 +1506,8 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		}()
 		_, err = f.WriteString(data + append)
 	} else {
-		obj1, err := rootFs.NewObject(src)
+		var obj1 fs.Object
+		obj1, err = rootFs.NewObject(src)
 		if err != nil {
 			return err
 		}
@@ -1632,15 +1639,13 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	} else {
-		if f.Features().UnWrap != nil {
-			cfs, ok := f.Features().UnWrap().(*cache.Fs)
-			if ok {
-				return cfs, nil
-			}
+	}
+	if f.Features().UnWrap != nil {
+		cfs, ok := f.Features().UnWrap().(*cache.Fs)
+		if ok {
+			return cfs, nil
 		}
 	}
-
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/cache_test.go

@@ -15,7 +15,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestCache:",
-		NilObject:  (*cache.Object)(nil),
+		RemoteName:                   "TestCache:",
+		NilObject:                    (*cache.Object)(nil),
+		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 	})
 }

backend/cache/storage_persistent.go

@@ -398,7 +398,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
+		err = bucket.Put([]byte(cachedObject.Name), encoded)
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -809,7 +809,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), []byte(encoded))
+		err = bucket.Put([]byte(destPath), encoded)
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -1023,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1049,7 +1049,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), []byte(encoded))
+			err = bucket.Put([]byte(destPath), encoded)
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}

backend/crypt/cipher.go

@@ -144,7 +144,6 @@ type cipher struct {
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
-	passCorrupted  bool
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -164,11 +163,6 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
-// Set to pass corrupted blocks
-func (c *cipher) setPassCorrupted(passCorrupted bool) {
-	c.passCorrupted = passCorrupted
-}
-
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
@@ -469,7 +463,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(rune(newRune))
+			_, _ = result.WriteRune(newRune)
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -754,7 +748,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retreive the nonce
+	// retrieve the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil
@@ -828,10 +822,7 @@ func (fh *decrypter) fillBuffer() (err error) {
 		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
-		if !fh.c.passCorrupted {
-			return ErrorEncryptedBadBlock
-		}
-		fs.Errorf(nil, "passing corrupted block")
+		return ErrorEncryptedBadBlock
 	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize

backend/crypt/crypt.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 )
 
+// Globals
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -79,15 +80,6 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
-		}, {
-			Name: "pass_corrupted_blocks",
-			Help: `Pass through corrupted blocks to the output.
-
-This is for debugging corruption problems in crypt - it shouldn't be needed normally.
-`,
-			Default:  false,
-			Hide:     fs.OptionHideConfigurator,
-			Advanced: true,
 		}},
 	})
 }
@@ -116,7 +108,6 @@ func newCipherForConfig(opt *Options) (Cipher, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make cipher")
 	}
-	cipher.setPassCorrupted(opt.PassCorruptedBlocks)
 	return cipher, nil
 }
 
@@ -131,7 +122,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -178,23 +169,10 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
+		SetTier:                 true,
+		GetTier:                 true,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
-	doChangeNotify := wrappedFs.Features().ChangeNotify
-	if doChangeNotify != nil {
-		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
-			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-				decrypted, err := f.DecryptFileName(path)
-				if err != nil {
-					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-					return
-				}
-				notifyFunc(decrypted, entryType)
-			}
-			doChangeNotify(wrappedNotifyFunc, pollInterval)
-		}
-	}
-
 	return f, err
 }
 
@@ -206,12 +184,12 @@ type Options struct {
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ShowMapping             bool   `config:"show_mapping"`
-	PassCorruptedBlocks     bool   `config:"pass_corrupted_blocks"`
 }
 
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
+	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
@@ -554,6 +532,16 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -565,7 +553,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 }
 
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calcuates the hash given by HashType on the fly
+// src with it, and calculates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
@@ -626,6 +614,75 @@ func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr
 	return m.Sums()[hashType], nil
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	out := make([]fs.Directory, len(dirs))
+	for i, dir := range dirs {
+		out[i] = fs.NewDirCopy(dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
+	}
+	return do(out)
+}
+
+// DirCacheFlush resets the directory cache - used in testing
+// as an optional interface
+func (f *Fs) DirCacheFlush() {
+	do := f.Fs.Features().DirCacheFlush
+	if do != nil {
+		do()
+	}
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(remote string) (string, error) {
+	do := f.Fs.Features().PublicLink
+	if do == nil {
+		return "", errors.New("PublicLink not supported")
+	}
+	o, err := f.NewObject(remote)
+	if err != nil {
+		// assume it is a directory
+		return do(f.cipher.EncryptDirName(remote))
+	}
+	return do(o.(*Object).Object.Remote())
+}
+
+// ChangeNotify calls the passed function with a path
+// that has had changes. If the implementation
+// uses polling, it should adhere to the given interval.
+func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+	do := f.Fs.Features().ChangeNotify
+	if do == nil {
+		return
+	}
+	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+		fs.Logf(f, "path %q entryType %d", path, entryType)
+		var (
+			err       error
+			decrypted string
+		)
+		switch entryType {
+		case fs.EntryDirectory:
+			decrypted, err = f.cipher.DecryptDirName(path)
+		case fs.EntryObject:
+			decrypted, err = f.cipher.DecryptFileName(path)
+		default:
+			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
+			return
+		}
+		if err != nil {
+			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+			return
+		}
+		notifyFunc(decrypted, entryType)
+	}
+	do(wrappedNotifyFunc, pollIntervalChan)
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -784,6 +841,34 @@ func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	do, ok := o.Object.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// SetTier performs changing storage tier of the Object if
+// multiple storage classes supported
+func (o *Object) SetTier(tier string) error {
+	do, ok := o.Object.(fs.SetTierer)
+	if !ok {
+		return errors.New("crypt: underlying remote does not support SetTier")
+	}
+	return do.SetTier(tier)
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *Object) GetTier() string {
+	do, ok := o.Object.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -797,7 +882,15 @@ var (
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.Wrapper         = (*Fs)(nil)
+	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+	_ fs.SetTierer       = (*Object)(nil)
+	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_test.go

@@ -21,8 +21,9 @@ func TestIntegration(t *testing.T) {
 		t.Skip("Skipping as -remote not set")
 	}
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: *fstest.RemoteName,
-		NilObject:  (*crypt.Object)(nil),
+		RemoteName:                   *fstest.RemoteName,
+		NilObject:                    (*crypt.Object)(nil),
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -42,6 +43,7 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -61,6 +63,7 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -80,6 +83,7 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters: true,
+		SkipBadWindowsCharacters:     true,
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }

backend/drive/drive.go

@@ -1,7 +1,4 @@
 // Package drive interfaces with the Google Drive object storage system
-
-// +build go1.9
-
 package drive
 
 // FIXME need to deal with some corner cases
@@ -21,6 +18,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -185,10 +183,10 @@ func init() {
 		},
 		Options: []fs.Option{{
 			Name: config.ConfigClientID,
-			Help: "Google Application Client Id\nLeave blank normally.",
+			Help: "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance.",
 		}, {
 			Name: config.ConfigClientSecret,
-			Help: "Google Application Client Secret\nLeave blank normally.",
+			Help: "Google Application Client Secret\nSetting your own is recommended.",
 		}, {
 			Name: "scope",
 			Help: "Scope that rclone should use when requesting access from drive.",
@@ -239,6 +237,22 @@ func init() {
 			Default:  false,
 			Help:     "Skip google documents in all listings.\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
+		}, {
+			Name:    "skip_checksum_gphotos",
+			Default: false,
+			Help: `Skip MD5 checksum on Google photos and videos only.
+
+Use this if you get checksum errors when transferring Google photos or
+videos.
+
+Setting this flag will cause Google photos and videos to return a
+blank MD5 checksum.
+
+Google photos are identifed by being in the "photos" space.
+
+Corrupted checksums are caused by Google modifying the image/video but
+not updating the checksum.`,
+			Advanced: true,
 		}, {
 			Name:    "shared_with_me",
 			Default: false,
@@ -395,6 +409,7 @@ type Options struct {
 	AuthOwnerOnly             bool          `config:"auth_owner_only"`
 	UseTrash                  bool          `config:"use_trash"`
 	SkipGdocs                 bool          `config:"skip_gdocs"`
+	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool          `config:"shared_with_me"`
 	TrashedOnly               bool          `config:"trashed_only"`
 	Extensions                string        `config:"formats"`
@@ -425,7 +440,7 @@ type Fs struct {
 	client           *http.Client       // authorized client
 	rootFolderID     string             // the id of the root folder
 	dirCache         *dircache.DirCache // Map of directory path to directory id
-	pacer            *pacer.Pacer       // To pace the API calls
+	pacer            *fs.Pacer          // To pace the API calls
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
@@ -481,7 +496,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whehter a given err rates being retried
+// shouldRetry determines whether a given err rates being retried
 func shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
@@ -614,6 +629,9 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, inc
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
+	if f.opt.SkipChecksumGphotos {
+		fields += ",spaces"
+	}
 
 	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
 
@@ -675,28 +693,33 @@ func isPowerOfTwo(x int64) bool {
 }
 
 // add a charset parameter to all text/* MIME types
-func fixMimeType(mimeType string) string {
-	mediaType, param, err := mime.ParseMediaType(mimeType)
+func fixMimeType(mimeTypeIn string) string {
+	if mimeTypeIn == "" {
+		return ""
+	}
+	mediaType, param, err := mime.ParseMediaType(mimeTypeIn)
 	if err != nil {
-		return mimeType
+		return mimeTypeIn
 	}
-	if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
+	mimeTypeOut := mimeTypeIn
+	if strings.HasPrefix(mediaType, "text/") && param["charset"] == "" {
 		param["charset"] = "utf-8"
-		mimeType = mime.FormatMediaType(mediaType, param)
+		mimeTypeOut = mime.FormatMediaType(mediaType, param)
 	}
-	return mimeType
+	if mimeTypeOut == "" {
+		panic(errors.Errorf("unable to fix MIME type %q", mimeTypeIn))
+	}
+	return mimeTypeOut
 }
-func fixMimeTypeMap(m map[string][]string) map[string][]string {
-	for _, v := range m {
+func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
+	out = make(map[string][]string, len(in))
+	for k, v := range in {
 		for i, mt := range v {
-			fixed := fixMimeType(mt)
-			if fixed == "" {
-				panic(errors.Errorf("unable to fix MIME type %q", mt))
-			}
-			v[i] = fixed
+			v[i] = fixMimeType(mt)
 		}
+		out[fixMimeType(k)] = v
 	}
-	return m
+	return out
 }
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
@@ -788,8 +811,8 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer(opt *Options) *pacer.Pacer {
-	return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
+func newPacer(opt *Options) *fs.Pacer {
+	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
@@ -862,7 +885,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -901,6 +924,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: true,
 	}).Fill(f)
 
 	// Create a new authorized Drive client.
@@ -995,6 +1019,15 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 
 // newRegularObject creates a fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
+	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
+	if f.opt.SkipChecksumGphotos {
+		for _, space := range info.Spaces {
+			if space == "photos" {
+				info.Md5Checksum = ""
+				break
+			}
+		}
+	}
 	return &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, info.Id),
@@ -1339,17 +1372,46 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
+// listREntry is a task to be executed by a litRRunner
+type listREntry struct {
+	id, path string
+}
+
+// listRSlices is a helper struct to sort two slices at once
+type listRSlices struct {
+	dirs  []string
+	paths []string
+}
+
+func (s listRSlices) Sort() {
+	sort.Sort(s)
+}
+
+func (s listRSlices) Len() int {
+	return len(s.dirs)
+}
+
+func (s listRSlices) Swap(i, j int) {
+	s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
+	s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
+}
+
+func (s listRSlices) Less(i, j int) bool {
+	return s.dirs[i] < s.dirs[j]
+}
+
 // listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
 //
-// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
+// In each cycle it will read up to grouping entries from the in channel without blocking.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
+func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
 	var dirs []string
+	var paths []string
 
 	for dir := range in {
-		dirs = append(dirs[:0], dir)
-		wait := time.After(10 * time.Millisecond)
+		dirs = append(dirs[:0], dir.id)
+		paths = append(paths[:0], dir.path)
 	waitloop:
 		for i := 1; i < grouping; i++ {
 			select {
@@ -1357,31 +1419,32 @@ func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error,
 				if !ok {
 					break waitloop
 				}
-				dirs = append(dirs, d)
-			case <-wait:
-				break waitloop
+				dirs = append(dirs, d.id)
+				paths = append(paths, d.path)
+			default:
 			}
 		}
+		listRSlices{dirs, paths}.Sort()
 		var iErr error
 		_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
-			parentPath := ""
-			if len(item.Parents) > 0 {
-				p, ok := f.dirCache.GetInv(item.Parents[0])
-				if ok {
-					parentPath = p
+			for _, parent := range item.Parents {
+				// only handle parents that are in the requested dirs list
+				i := sort.SearchStrings(dirs, parent)
+				if i == len(dirs) || dirs[i] != parent {
+					continue
+				}
+				remote := path.Join(paths[i], item.Name)
+				entry, err := f.itemToDirEntry(remote, item)
+				if err != nil {
+					iErr = err
+					return true
 				}
-			}
-			remote := path.Join(parentPath, item.Name)
-			entry, err := f.itemToDirEntry(remote, item)
-			if err != nil {
-				iErr = err
-				return true
-			}
 
-			err = cb(entry)
-			if err != nil {
-				iErr = err
-				return true
+				err = cb(entry)
+				if err != nil {
+					iErr = err
+					return true
+				}
 			}
 			return false
 		})
@@ -1432,30 +1495,44 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	if directoryID == "root" {
+		var info *drive.File
+		err = f.pacer.CallNoRetry(func() (bool, error) {
+			info, err = f.svc.Files.Get("root").
+				Fields("id").
+				SupportsTeamDrives(f.isTeamDrive).
+				Do()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return err
+		}
+		directoryID = info.Id
+	}
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
-	in := make(chan string, inputBuffer)
+	in := make(chan listREntry, inputBuffer)
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
-	overfflow := []string{}
+	overflow := []listREntry{}
 
 	cb := func(entry fs.DirEntry) error {
 		mu.Lock()
 		defer mu.Unlock()
 		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
 			select {
-			case in <- d.ID():
+			case in <- listREntry{d.ID(), d.Remote()}:
 				wg.Add(1)
 			default:
-				overfflow = append(overfflow, d.ID())
+				overflow = append(overflow, listREntry{d.ID(), d.Remote()})
 			}
 		}
 		return list.Add(entry)
 	}
 
 	wg.Add(1)
-	in <- directoryID
+	in <- listREntry{directoryID, dir}
 
 	for i := 0; i < fs.Config.Checkers; i++ {
 		go f.listRRunner(&wg, in, out, cb, grouping)
@@ -1464,18 +1541,18 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 		// wait until the all directories are processed
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
-		for len(overfflow) > 0 {
+		for len(overflow) > 0 {
 			mu.Lock()
-			l := len(overfflow)
-			// only fill half of the channel to prevent entries beeing put into overfflow again
+			l := len(overflow)
+			// only fill half of the channel to prevent entries beeing put into overflow again
 			if l > inputBuffer/2 {
 				l = inputBuffer / 2
 			}
 			wg.Add(l)
-			for _, d := range overfflow[:l] {
+			for _, d := range overflow[:l] {
 				in <- d
 			}
-			overfflow = overfflow[l:]
+			overflow = overflow[l:]
 			mu.Unlock()
 
 			// wait again for the completion of all directories
@@ -1666,14 +1743,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.rmdir(srcDir.ID(), true)
 		if err != nil {
-			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1792,16 +1869,24 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		remote = remote[:len(remote)-len(ext)]
 	}
 
+	// Look to see if there is an existing object
+	existingObject, _ := f.NewObject(remote)
+
 	createInfo, err := f.createFileInfo(remote, src.ModTime())
 	if err != nil {
 		return nil, err
 	}
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Copy(srcObj.id, createInfo).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
 			Do()
 		return shouldRetry(err)
@@ -1809,7 +1894,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.newObjectWithInfo(remote, info)
+	newObject, err := f.newObjectWithInfo(remote, info)
+	if err != nil {
+		return nil, err
+	}
+	if existingObject != nil {
+		err = existingObject.Remove()
+		if err != nil {
+			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
+		}
+	}
+	return newObject, nil
 }
 
 // Purge deletes all the files and the container
@@ -1935,6 +2030,11 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
@@ -1942,7 +2042,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			Do()
 		return shouldRetry(err)
 	})
@@ -1953,6 +2053,20 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, info)
 }
 
+// ShouldSupportTeamDrives returns the request should support TeamDrives
+func (f *Fs) ShouldSupportTeamDrives(src fs.Object) (bool, error) {
+	srcIsTeamDrive := false
+	if srcFs, ok := src.Fs().(*Fs); ok {
+		srcIsTeamDrive = srcFs.isTeamDrive
+	}
+
+	if f.isTeamDrive {
+		return true, nil
+	}
+
+	return srcIsTeamDrive, nil
+}
+
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(remote string) (link string, err error) {
 	id, err := f.dirCache.FindDir(remote, false)
@@ -2092,7 +2206,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // ChangeNotify calls the passed function with a path that has had changes.
 // If the implementation uses polling, it should adhere to the given interval.
 //
-// Automatically restarts itself in case of unexpected behaviour of the remote.
+// Automatically restarts itself in case of unexpected behavior of the remote.
 //
 // Close the returned channel to stop being notified.
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
@@ -2199,11 +2313,13 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPage
 
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
-					if parentPath, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
-						// and append the drive file name to compute the full file name
-						newPath := path.Join(parentPath, change.File.Name)
-						// this will now clear the actual file too
-						pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
+					for _, parent := range change.File.Parents {
+						if parentPath, ok := f.dirCache.GetInv(parent); ok {
+							// and append the drive file name to compute the full file name
+							newPath := path.Join(parentPath, change.File.Name)
+							// this will now clear the actual file too
+							pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
+						}
 					}
 				} else { // a true root object that is changed
 					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
@@ -2383,6 +2499,10 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
 		return req, nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
+	if o.bytes == 0 {
+		// Don't supply range requests for 0 length objects as they always fail
+		delete(req.Header, "Range")
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {
@@ -2586,6 +2706,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
+	if err != nil {
+		return err
+	}
 	switch newO := newO.(type) {
 	case *Object:
 		*o = *newO
@@ -2624,6 +2747,9 @@ func (o *documentObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.O
 	remote = remote[:len(remote)-o.extLen]
 
 	newO, err := o.fs.newObjectWithInfo(remote, info)
+	if err != nil {
+		return err
+	}
 	switch newO := newO.(type) {
 	case *documentObject:
 		*o = *newO

backend/drive/drive_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_test.go

@@ -1,7 +1,5 @@
 // Test Drive filesystem interface
 
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package drive

backend/drive/upload.go

@@ -8,8 +8,6 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
-// +build go1.9
-
 package drive
 
 import (
@@ -185,7 +183,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionaly on
+	// any other 2xx codes, so we parse it unconditionally on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err

backend/dropbox/dropbox.go

@@ -130,8 +130,8 @@ Any files larger than this will be uploaded in chunks of this size.
 Note that chunks are buffered in memory (one at a time) so rclone can
 deal with retries.  Setting this larger will increase the speed
 slightly (at most 10%% for 128MB in tests) at the cost of using more
-memory.  It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
-			Default:  fs.SizeSuffix(defaultChunkSize),
+memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
+			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
 			Name:     "impersonate",
@@ -160,7 +160,7 @@ type Fs struct {
 	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *pacer.Pacer   // To pace the API calls
+	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 }
 
@@ -209,12 +209,12 @@ func shouldRetry(err error) (bool, error) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
 			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
-			time.Sleep(time.Duration(e.RateLimitError.RetryAfter) * time.Second)
+			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
-	// Keep old behaviour for backward compatibility
-	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
+	// Keep old behavior for backward compatibility
+	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") || baseErrString == "" {
 		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
@@ -239,7 +239,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -273,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	config := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo

backend/ftp/ftp.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -45,6 +46,11 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
+			}, {
+				Name:     "concurrency",
+				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
+				Default:  0,
+				Advanced: true,
 			},
 		},
 	})
@@ -52,10 +58,11 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host string `config:"host"`
-	User string `config:"user"`
-	Pass string `config:"pass"`
-	Port string `config:"port"`
+	Host        string `config:"host"`
+	User        string `config:"user"`
+	Pass        string `config:"pass"`
+	Port        string `config:"port"`
+	Concurrency int    `config:"concurrency"`
 }
 
 // Fs represents a remote FTP server
@@ -70,6 +77,7 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
+	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -128,6 +136,9 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		f.tokens.Get()
+	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -147,6 +158,9 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		defer f.tokens.Put()
+	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -166,7 +180,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	f.poolMu.Unlock()
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
@@ -198,6 +212,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
+		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,

backend/googlecloudstorage/googlecloudstorage.go

@@ -1,7 +1,4 @@
 // Package googlecloudstorage provides an interface to Google Cloud Storage
-
-// +build go1.9
-
 package googlecloudstorage
 
 /*
@@ -16,6 +13,7 @@ FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 erro
 */
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -45,6 +43,8 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
+
+	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
 )
 
@@ -144,6 +144,22 @@ func init() {
 				Value: "publicReadWrite",
 				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",
 			}},
+		}, {
+			Name: "bucket_policy_only",
+			Help: `Access checks should use bucket-level IAM policies.
+
+If you want to upload objects to a bucket with Bucket Policy Only set
+then you will need to set this.
+
+When it is set, rclone:
+
+- ignores ACLs set on buckets
+- ignores ACLs set on objects
+- creates buckets with Bucket Policy Only set
+
+Docs: https://cloud.google.com/storage/docs/bucket-policy-only
+`,
+			Default: false,
 		}, {
 			Name: "location",
 			Help: "Location for the newly created buckets.",
@@ -162,21 +178,36 @@ func init() {
 			}, {
 				Value: "asia-east1",
 				Help:  "Taiwan.",
+			}, {
+				Value: "asia-east2",
+				Help:  "Hong Kong.",
 			}, {
 				Value: "asia-northeast1",
 				Help:  "Tokyo.",
+			}, {
+				Value: "asia-south1",
+				Help:  "Mumbai.",
 			}, {
 				Value: "asia-southeast1",
 				Help:  "Singapore.",
 			}, {
 				Value: "australia-southeast1",
 				Help:  "Sydney.",
+			}, {
+				Value: "europe-north1",
+				Help:  "Finland.",
 			}, {
 				Value: "europe-west1",
 				Help:  "Belgium.",
 			}, {
 				Value: "europe-west2",
 				Help:  "London.",
+			}, {
+				Value: "europe-west3",
+				Help:  "Frankfurt.",
+			}, {
+				Value: "europe-west4",
+				Help:  "Netherlands.",
 			}, {
 				Value: "us-central1",
 				Help:  "Iowa.",
@@ -189,6 +220,9 @@ func init() {
 			}, {
 				Value: "us-west1",
 				Help:  "Oregon.",
+			}, {
+				Value: "us-west2",
+				Help:  "California.",
 			}},
 		}, {
 			Name: "storage_class",
@@ -223,6 +257,7 @@ type Options struct {
 	ServiceAccountCredentials string `config:"service_account_credentials"`
 	ObjectACL                 string `config:"object_acl"`
 	BucketACL                 string `config:"bucket_acl"`
+	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
 	Location                  string `config:"location"`
 	StorageClass              string `config:"storage_class"`
 }
@@ -238,7 +273,7 @@ type Fs struct {
 	bucket     string           // the bucket we are working on
 	bucketOKMu sync.Mutex       // mutex to protect bucket OK
 	bucketOK   bool             // true if we have created the bucket
-	pacer      *pacer.Pacer     // To pace the API calls
+	pacer      *fs.Pacer        // To pace the API calls
 }
 
 // Object describes a storage object
@@ -282,7 +317,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whehter a given err rates being retried
+// shouldRetry determines whether a given err rates being retried
 func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
@@ -330,7 +365,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 
@@ -363,7 +398,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
+			ctx := context.Background()
+			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
+			}
 		}
 	}
 
@@ -377,7 +416,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		opt:    *opt,
-		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
+		pacer:  fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -691,8 +730,19 @@ func (f *Fs) Mkdir(dir string) (err error) {
 		Location:     f.opt.Location,
 		StorageClass: f.opt.StorageClass,
 	}
+	if f.opt.BucketPolicyOnly {
+		bucket.IamConfiguration = &storage.BucketIamConfiguration{
+			BucketPolicyOnly: &storage.BucketIamConfigurationBucketPolicyOnly{
+				Enabled: true,
+			},
+		}
+	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
+		insertBucket := f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket)
+		if !f.opt.BucketPolicyOnly {
+			insertBucket.PredefinedAcl(f.opt.BucketACL)
+		}
+		_, err = insertBucket.Do()
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -958,7 +1008,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
+		insertObject := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
+		if !o.fs.opt.BucketPolicyOnly {
+			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
+		}
+		newObject, err = insertObject.Do()
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,7 +1,5 @@
 // Test GoogleCloudStorage filesystem interface
 
-// +build go1.9
-
 package googlecloudstorage_test
 
 import (

backend/googlecloudstorage/googlecloudstorage_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package googlecloudstorage

backend/http/http.go

@@ -6,6 +6,7 @@ package http
 
 import (
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -40,7 +41,26 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "https://example.com",
 				Help:  "Connect to example.com",
+			}, {
+				Value: "https://user:pass@example.com",
+				Help:  "Connect to example.com using a username and password",
 			}},
+		}, {
+			Name: "no_slash",
+			Help: `Set this if the site doesn't end directories with /
+
+Use this if your target website does not use / on the end of
+directories.
+
+A / on the end of a path is how rclone normally tells the difference
+between files and directories.  If this flag is set, then rclone will
+treat all files with Content-Type: text/html as directories and read
+URLs from them rather than downloading them.
+
+Note that this may cause rclone to confuse genuine HTML files with
+directories.`,
+			Default:  false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -49,6 +69,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
+	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -248,7 +269,7 @@ func parseName(base *url.URL, name string) (string, error) {
 	}
 	// calculate the name relative to the base
 	name = u.Path[len(base.Path):]
-	// musn't be empty
+	// mustn't be empty
 	if name == "" {
 		return "", errNameIsEmpty
 	}
@@ -267,14 +288,20 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var walk func(*html.Node)
+	var (
+		walk func(*html.Node)
+		seen = make(map[string]struct{})
+	)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						names = append(names, name)
+						if _, found := seen[name]; !found {
+							names = append(names, name)
+							seen[name] = struct{}{}
+						}
 					}
 					break
 				}
@@ -299,14 +326,16 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil && res.StatusCode == http.StatusNotFound {
-		return nil, fs.ErrorDirNotFound
+	if err == nil {
+		defer fs.CheckClose(res.Body, &err)
+		if res.StatusCode == http.StatusNotFound {
+			return nil, fs.ErrorDirNotFound
+		}
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
-	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -350,11 +379,16 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			if err = file.stat(); err != nil {
+			switch err = file.stat(); err {
+			case nil:
+				entries = append(entries, file)
+			case fs.ErrorNotAFile:
+				// ...found a directory not a file
+				dir := fs.NewDir(remote, timeUnset)
+				entries = append(entries, dir)
+			default:
 				fs.Debugf(remote, "skipping because of error: %v", err)
-				continue
 			}
-			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -430,6 +464,16 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	// If NoSlash is set then check ContentType to see if it is a directory
+	if o.fs.opt.NoSlash {
+		mediaType, _, err := mime.ParseMediaType(o.contentType)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
+		}
+		if mediaType == "text/html" {
+			return fs.ErrorNotAFile
+		}
+	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.8
-
 package http
 
 import (
@@ -65,7 +63,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs) {
+func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -93,15 +91,29 @@ func testListRoot(t *testing.T, f fs.Fs) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	assert.Equal(t, int64(7), e.Size())
-	_, ok = e.(*Object)
-	assert.True(t, ok)
+	if noSlash {
+		assert.Equal(t, int64(-1), e.Size())
+		_, ok = e.(fs.Directory)
+		assert.True(t, ok)
+	} else {
+		assert.Equal(t, int64(41), e.Size())
+		_, ok = e.(*Object)
+		assert.True(t, ok)
+	}
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f)
+	testListRoot(t, f, false)
+}
+
+func TestListRootNoSlash(t *testing.T) {
+	f, tidy := prepare(t)
+	f.(*Fs).opt.NoSlash = true
+	defer tidy()
+
+	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -194,7 +206,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f)
+	testListRoot(t, f, false)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-potato
+<a href="two.html/file.txt">file.txt</a>

backend/hubic/hubic.go

@@ -9,8 +9,10 @@ package hubic
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/swift"
@@ -124,7 +126,9 @@ func (f *Fs) getCredentials() (err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return errors.Errorf("failed to get credentials: %s", resp.Status)
+		body, _ := ioutil.ReadAll(resp.Body)
+		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
+		return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
 	}
 	decoder := json.NewDecoder(resp.Body)
 	var result credentials

backend/hubic/hubic_test.go

@@ -11,7 +11,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestHubic:",
-		NilObject:  (*hubic.Object)(nil),
+		RemoteName:          "TestHubic:",
+		NilObject:           (*hubic.Object)(nil),
+		SkipFsCheckWrap:     true,
+		SkipObjectCheckWrap: true,
 	})
 }

backend/jottacloud/api/types.go

@@ -314,3 +314,9 @@ type UploadResponse struct {
 	Deleted   interface{} `json:"deleted"`
 	Mime      string      `json:"mime"`
 }
+
+// DeviceRegistrationResponse is the response to registering a device
+type DeviceRegistrationResponse struct {
+	ClientID     string `json:"client_id"`
+	ClientSecret string `json:"client_secret"`
+}

backend/jottacloud/jottacloud.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -40,15 +41,19 @@ const (
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Sync"
+	defaultMountpoint           = "Sync" // nolint
 	rootURL                     = "https://www.jottacloud.com/jfs/"
 	apiURL                      = "https://api.jottacloud.com/files/v1/"
 	baseURL                     = "https://www.jottacloud.com/"
 	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
+	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
 	cachePrefix                 = "rclone-jcmd5-"
 	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
 	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
 	configUsername              = "user"
+	configClientID              = "client_id"
+	configClientSecret          = "client_secret"
+	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 )
 
 var (
@@ -58,9 +63,7 @@ var (
 			AuthURL:  tokenURL,
 			TokenURL: tokenURL,
 		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
+		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
 )
 
@@ -79,6 +82,55 @@ func init() {
 				}
 			}
 
+			srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+			// ask if we should create a device specifc token: https://github.com/ncw/rclone/issues/2995
+			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+			if config.Confirm() {
+				// random generator to generate random device names
+				seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+				randonDeviceNamePartLength := 21
+				randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
+				for i := range randomDeviceNamePart {
+					randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
+				}
+				randomDeviceName := "rclone-" + string(randomDeviceNamePart)
+				fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
+
+				values := url.Values{}
+				values.Set("device_id", randomDeviceName)
+
+				// all information comes from https://github.com/ttyridal/aiojotta/wiki/Jotta-protocol-3.-Authentication#token-authentication
+				opts := rest.Opts{
+					Method:       "POST",
+					RootURL:      registerURL,
+					ContentType:  "application/x-www-form-urlencoded",
+					ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
+					Parameters:   values,
+				}
+
+				var deviceRegistration api.DeviceRegistrationResponse
+				_, err := srv.CallJSON(&opts, nil, &deviceRegistration)
+				if err != nil {
+					log.Fatalf("Failed to register device: %v", err)
+				}
+
+				m.Set(configClientID, deviceRegistration.ClientID)
+				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+			}
+
+			clientID, ok := m.Get(configClientID)
+			if !ok {
+				clientID = rcloneClientID
+			}
+			clientSecret, ok := m.Get(configClientSecret)
+			if !ok {
+				clientSecret = rcloneEncryptedClientSecret
+			}
+			oauthConfig.ClientID = clientID
+			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
 			username, ok := m.Get(configUsername)
 			if !ok {
 				log.Fatalf("No username defined")
@@ -86,7 +138,6 @@ func init() {
 			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
 
 			// prepare out token request with username and password
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
 			values := url.Values{}
 			values.Set("grant_type", "PASSWORD")
 			values.Set("password", password)
@@ -103,7 +154,7 @@ func init() {
 			var jsonToken api.TokenJSON
 			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
 			if err != nil {
-				// if 2fa is enabled the first request is expected to fail. we'lls do another request with the 2fa code as an additional http header
+				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
 				if resp != nil {
 					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
 						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
@@ -163,7 +214,7 @@ func init() {
 			Advanced: true,
 		}, {
 			Name:     "upload_resume_limit",
-			Help:     "Files bigger than this can be resumed if the upload failes.",
+			Help:     "Files bigger than this can be resumed if the upload fail's.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
 		}},
@@ -190,7 +241,7 @@ type Fs struct {
 	endpointURL  string
 	srv          *rest.Client
 	apiSrv       *rest.Client
-	pacer        *pacer.Pacer
+	pacer        *fs.Pacer
 	tokenRenewer *oauthutil.Renew // renew the token on expiry
 }
 
@@ -361,7 +412,7 @@ func grantTypeFilter(req *http.Request) {
 		}
 		_ = req.Body.Close()
 
-		// make the refesh token upper case
+		// make the refresh token upper case
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
@@ -381,6 +432,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = rcloneClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = rcloneEncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+	// add jottacloud to the long list of sites that don't follow the oauth spec correctly
+	oauth2.RegisterBrokenAuthHeaderProvider("https://www.jottacloud.com/")
+
 	// the oauth client for the api servers needs
 	// a filter to fix the grant_type issues (see above)
 	baseClient := fshttp.NewClient(fs.Config)
@@ -403,7 +468,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:    *opt,
 		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
 		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
-		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -769,7 +834,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copys or moves directories or files depending on the mthod parameter
+// copyOrMoves copies or moves directories or files depending on the method parameter
 func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -1006,7 +1071,7 @@ func (o *Object) MimeType() string {
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.hasMetaData = true
-	o.size = int64(info.Size)
+	o.size = info.Size
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
 	o.modTime = time.Time(info.ModifiedAt)
@@ -1080,7 +1145,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
 	// we need a MD5
 	md5Hasher := md5.New()
-	// use the teeReader to write to the local file AND caclulate the MD5 while doing so
+	// use the teeReader to write to the local file AND calculate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
 
 	// nothing to clean up by default
@@ -1212,7 +1277,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 
 		// finally update the meta data
 		o.hasMetaData = true
-		o.size = int64(result.Bytes)
+		o.size = result.Bytes
 		o.md5 = result.Md5
 		o.modTime = time.Unix(result.Modified/1000, 0)
 	} else {

backend/jottacloud/replace.go

@@ -2,7 +2,7 @@
 Translate file names for JottaCloud adapted from OneDrive
 
 
-The following characters are JottaClous reserved characters, and can't
+The following characters are JottaCloud reserved characters, and can't
 be used in JottaCloud folder and file names.
 
   jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"

backend/koofr/koofr.go

@@ -0,0 +1,589 @@
+package koofr
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/hash"
+
+	httpclient "github.com/koofr/go-httpclient"
+	koofrclient "github.com/koofr/go-koofrclient"
+)
+
+// Register Fs with rclone
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "koofr",
+		Description: "Koofr",
+		NewFs:       NewFs,
+		Options: []fs.Option{
+			{
+				Name:     "endpoint",
+				Help:     "The Koofr API endpoint to use",
+				Default:  "https://app.koofr.net",
+				Required: true,
+				Advanced: true,
+			}, {
+				Name:     "mountid",
+				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
+				Required: false,
+				Default:  "",
+				Advanced: true,
+			}, {
+				Name:     "user",
+				Help:     "Your Koofr user name",
+				Required: true,
+			}, {
+				Name:       "password",
+				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
+				IsPassword: true,
+				Required:   true,
+			},
+		},
+	})
+}
+
+// Options represent the configuration of the Koofr backend
+type Options struct {
+	Endpoint string `config:"endpoint"`
+	MountID  string `config:"mountid"`
+	User     string `config:"user"`
+	Password string `config:"password"`
+}
+
+// A Fs is a representation of a remote Koofr Fs
+type Fs struct {
+	name     string
+	mountID  string
+	root     string
+	opt      Options
+	features *fs.Features
+	client   *koofrclient.KoofrClient
+}
+
+// An Object on the remote Koofr Fs
+type Object struct {
+	fs     *Fs
+	remote string
+	info   koofrclient.FileInfo
+}
+
+func base(pth string) string {
+	rv := path.Base(pth)
+	if rv == "" || rv == "." {
+		rv = "/"
+	}
+	return rv
+}
+
+func dir(pth string) string {
+	rv := path.Dir(pth)
+	if rv == "" || rv == "." {
+		rv = "/"
+	}
+	return rv
+}
+
+// String returns a string representation of the remote Object
+func (o *Object) String() string {
+	return o.remote
+}
+
+// Remote returns the remote path of the Object, relative to Fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the Object
+func (o *Object) ModTime() time.Time {
+	return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000)
+}
+
+// Size return the size of the Object in bytes
+func (o *Object) Size() int64 {
+	return o.info.Size
+}
+
+// Fs returns a reference to the Koofr Fs containing the Object
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns an MD5 hash of the Object
+func (o *Object) Hash(typ hash.Type) (string, error) {
+	if typ == hash.MD5 {
+		return o.info.Hash, nil
+	}
+	return "", nil
+}
+
+// fullPath returns full path of the remote Object (including Fs root)
+func (o *Object) fullPath() string {
+	return o.fs.fullPath(o.remote)
+}
+
+// Storable returns true if the Object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime is not supported
+func (o *Object) SetModTime(mtime time.Time) error {
+	return nil
+}
+
+// Open opens the Object for reading
+func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+	var sOff, eOff int64 = 0, -1
+
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			sOff = x.Offset
+		case *fs.RangeOption:
+			sOff = x.Start
+			eOff = x.End
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	if sOff == 0 && eOff < 0 {
+		return o.fs.client.FilesGet(o.fs.mountID, o.fullPath())
+	}
+	if sOff < 0 {
+		sOff = o.Size() - eOff
+		eOff = o.Size()
+	}
+	if eOff > o.Size() {
+		eOff = o.Size()
+	}
+	span := &koofrclient.FileSpan{
+		Start: sOff,
+		End:   eOff,
+	}
+	return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span)
+}
+
+// Update updates the Object contents
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	putopts := &koofrclient.PutFilter{
+		ForceOverwrite:    true,
+		NoRename:          true,
+		IgnoreNonExisting: true,
+	}
+	fullPath := o.fullPath()
+	dirPath := dir(fullPath)
+	name := base(fullPath)
+	err := o.fs.mkdir(dirPath)
+	if err != nil {
+		return err
+	}
+	info, err := o.fs.client.FilesPutOptions(o.fs.mountID, dirPath, name, in, putopts)
+	if err != nil {
+		return err
+	}
+	o.info = *info
+	return nil
+}
+
+// Remove deletes the remote Object
+func (o *Object) Remove() error {
+	return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath())
+}
+
+// Name returns the name of the Fs
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root path of the Fs
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a string representation of the Fs
+func (f *Fs) String() string {
+	return "koofr:" + f.mountID + ":" + f.root
+}
+
+// Features returns the optional features supported by this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision denotes that setting modification times is not supported
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns a set of hashes are Provided by the Fs
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.MD5)
+}
+
+// fullPath constructs a full, absolute path from a Fs root relative path,
+func (f *Fs) fullPath(part string) string {
+	return path.Join("/", f.root, part)
+}
+
+// NewFs constructs a new filesystem given a root path and configuration options
+func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	pass, err := obscure.Reveal(opt.Password)
+	if err != nil {
+		return nil, err
+	}
+	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
+	basicAuth := fmt.Sprintf("Basic %s",
+		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
+	client.HTTPClient.Headers.Set("Authorization", basicAuth)
+	mounts, err := client.Mounts()
+	if err != nil {
+		return nil, err
+	}
+	f := &Fs{
+		name:   name,
+		root:   root,
+		opt:    *opt,
+		client: client,
+	}
+	f.features = (&fs.Features{
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		BucketBased:             false,
+		CanHaveEmptyDirectories: true,
+	}).Fill(f)
+	for _, m := range mounts {
+		if opt.MountID != "" {
+			if m.Id == opt.MountID {
+				f.mountID = m.Id
+				break
+			}
+		} else if m.IsPrimary {
+			f.mountID = m.Id
+			break
+		}
+	}
+	if f.mountID == "" {
+		if opt.MountID == "" {
+			return nil, errors.New("Failed to find primary mount")
+		}
+		return nil, errors.New("Failed to find mount " + opt.MountID)
+	}
+	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
+	if err == nil && rootFile.Type != "dir" {
+		f.root = dir(f.root)
+		err = fs.ErrorIsFile
+	} else {
+		err = nil
+	}
+	return f, err
+}
+
+// List returns a list of items in a directory
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return nil, translateErrorsDir(err)
+	}
+	entries = make([]fs.DirEntry, len(files))
+	for i, file := range files {
+		if file.Type == "dir" {
+			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
+		} else {
+			entries[i] = &Object{
+				fs:     f,
+				info:   file,
+				remote: path.Join(dir, file.Name),
+			}
+		}
+	}
+	return entries, nil
+}
+
+// NewObject creates a new remote Object for a given remote path
+func (f *Fs) NewObject(remote string) (obj fs.Object, err error) {
+	info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote))
+	if err != nil {
+		return nil, translateErrorsObject(err)
+	}
+	if info.Type == "dir" {
+		return nil, fs.ErrorNotAFile
+	}
+	return &Object{
+		fs:     f,
+		info:   info,
+		remote: remote,
+	}, nil
+}
+
+// Put updates a remote Object
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
+	putopts := &koofrclient.PutFilter{
+		ForceOverwrite:    true,
+		NoRename:          true,
+		IgnoreNonExisting: true,
+	}
+	fullPath := f.fullPath(src.Remote())
+	dirPath := dir(fullPath)
+	name := base(fullPath)
+	err = f.mkdir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	info, err := f.client.FilesPutOptions(f.mountID, dirPath, name, in, putopts)
+	if err != nil {
+		return nil, translateErrorsObject(err)
+	}
+	return &Object{
+		fs:     f,
+		info:   *info,
+		remote: src.Remote(),
+	}, nil
+}
+
+// PutStream updates a remote Object with a stream of unknown size
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(in, src, options...)
+}
+
+// isBadRequest is a predicate which holds true iff the error returned was
+// HTTP status 400
+func isBadRequest(err error) bool {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusBadRequest {
+			return true
+		}
+	}
+	return false
+}
+
+// translateErrorsDir translates koofr errors to rclone errors (for a dir
+// operation)
+func translateErrorsDir(err error) error {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusNotFound {
+			return fs.ErrorDirNotFound
+		}
+	}
+	return err
+}
+
+// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
+func translateErrorsObject(err error) error {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusNotFound {
+			return fs.ErrorObjectNotFound
+		}
+	}
+	return err
+}
+
+// mkdir creates a directory at the given remote path. Creates ancestors if
+// neccessary
+func (f *Fs) mkdir(fullPath string) error {
+	if fullPath == "/" {
+		return nil
+	}
+	info, err := f.client.FilesInfo(f.mountID, fullPath)
+	if err == nil && info.Type == "dir" {
+		return nil
+	}
+	err = translateErrorsDir(err)
+	if err != nil && err != fs.ErrorDirNotFound {
+		return err
+	}
+	dirs := strings.Split(fullPath, "/")
+	parent := "/"
+	for _, part := range dirs {
+		if part == "" {
+			continue
+		}
+		info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part))
+		if err != nil || info.Type != "dir" {
+			err = translateErrorsDir(err)
+			if err != nil && err != fs.ErrorDirNotFound {
+				return err
+			}
+			err = f.client.FilesNewFolder(f.mountID, parent, part)
+			if err != nil && !isBadRequest(err) {
+				return err
+			}
+		}
+		parent = path.Join(parent, part)
+	}
+	return nil
+}
+
+// Mkdir creates a directory at the given remote path. Creates ancestors if
+// necessary
+func (f *Fs) Mkdir(dir string) error {
+	fullPath := f.fullPath(dir)
+	return f.mkdir(fullPath)
+}
+
+// Rmdir removes an (empty) directory at the given remote path
+func (f *Fs) Rmdir(dir string) error {
+	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return translateErrorsDir(err)
+	}
+	if len(files) > 0 {
+		return fs.ErrorDirectoryNotEmpty
+	}
+	err = f.client.FilesDelete(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return translateErrorsDir(err)
+	}
+	return nil
+}
+
+// Copy copies a remote Object to the given path
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+	dstFullPath := f.fullPath(remote)
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return nil, fs.ErrorCantCopy
+	}
+	err = f.client.FilesCopy((src.(*Object)).fs.mountID,
+		(src.(*Object)).fs.fullPath((src.(*Object)).remote),
+		f.mountID, dstFullPath)
+	if err != nil {
+		return nil, fs.ErrorCantCopy
+	}
+	return f.NewObject(remote)
+}
+
+// Move moves a remote Object to the given path
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
+	srcObj := src.(*Object)
+	dstFullPath := f.fullPath(remote)
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return nil, fs.ErrorCantMove
+	}
+	err = f.client.FilesMove(srcObj.fs.mountID,
+		srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath)
+	if err != nil {
+		return nil, fs.ErrorCantMove
+	}
+	return f.NewObject(remote)
+}
+
+// DirMove moves a remote directory to the given path
+func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs := src.(*Fs)
+	srcFullPath := srcFs.fullPath(srcRemote)
+	dstFullPath := f.fullPath(dstRemote)
+	if srcFs.mountID == f.mountID && srcFullPath == dstFullPath {
+		return fs.ErrorDirExists
+	}
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return fs.ErrorCantDirMove
+	}
+	err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath)
+	if err != nil {
+		return fs.ErrorCantDirMove
+	}
+	return nil
+}
+
+// About reports space usage (with a MB precision)
+func (f *Fs) About() (*fs.Usage, error) {
+	mount, err := f.client.MountsDetails(f.mountID)
+	if err != nil {
+		return nil, err
+	}
+	return &fs.Usage{
+		Total:   fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024),
+		Used:    fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024),
+		Trashed: nil,
+		Other:   nil,
+		Free:    fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024),
+		Objects: nil,
+	}, nil
+}
+
+// Purge purges the complete Fs
+func (f *Fs) Purge() error {
+	err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath("")))
+	return err
+}
+
+// linkCreate is a Koofr API request for creating a public link
+type linkCreate struct {
+	Path string `json:"path"`
+}
+
+// link is a Koofr API response to creating a public link
+type link struct {
+	ID               string `json:"id"`
+	Name             string `json:"name"`
+	Path             string `json:"path"`
+	Counter          int64  `json:"counter"`
+	URL              string `json:"url"`
+	ShortURL         string `json:"shortUrl"`
+	Hash             string `json:"hash"`
+	Host             string `json:"host"`
+	HasPassword      bool   `json:"hasPassword"`
+	Password         string `json:"password"`
+	ValidFrom        int64  `json:"validFrom"`
+	ValidTo          int64  `json:"validTo"`
+	PasswordRequired bool   `json:"passwordRequired"`
+}
+
+// createLink makes a Koofr API call to create a public link
+func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) {
+	linkCreate := linkCreate{
+		Path: path,
+	}
+	linkData := link{}
+
+	request := httpclient.RequestData{
+		Method:         "POST",
+		Path:           "/api/v2/mounts/" + mountID + "/links",
+		ExpectedStatus: []int{http.StatusOK, http.StatusCreated},
+		ReqEncoding:    httpclient.EncodingJSON,
+		ReqValue:       linkCreate,
+		RespEncoding:   httpclient.EncodingJSON,
+		RespValue:      &linkData,
+	}
+
+	_, err := c.Request(&request)
+	if err != nil {
+		return nil, err
+	}
+	return &linkData, nil
+}
+
+// PublicLink creates a public link to the remote path
+func (f *Fs) PublicLink(remote string) (string, error) {
+	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
+	if err != nil {
+		return "", translateErrorsDir(err)
+	}
+	return linkData.ShortURL, nil
+}

backend/koofr/koofr_test.go

@@ -0,0 +1,14 @@
+package koofr_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKoofr:",
+	})
+}

backend/local/about_unix.go

@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
-	bs := int64(s.Bsize)
+	bs := int64(s.Bsize) // nolint: unconvert
 	usage := &fs.Usage{
 		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
 		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use

backend/local/local.go

@@ -225,10 +225,10 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// caseInsenstive returns whether the remote is case insensitive or not
+// caseInsensitive returns whether the remote is case insensitive or not
 func (f *Fs) caseInsensitive() bool {
 	// FIXME not entirely accurate since you can have case
-	// sensitive Fses on darwin and case insenstive Fses on linux.
+	// sensitive Fses on darwin and case insensitive Fses on linux.
 	// Should probably check but that would involve creating a
 	// file in the remote to be most accurate which probably isn't
 	// desirable.
@@ -288,7 +288,7 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
-		// Handle the odd case, that a symlink was specfied by name without the link suffix
+		// Handle the odd case, that a symlink was specified by name without the link suffix
 		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
 			return nil, fs.ErrorObjectNotFound
 		}
@@ -958,7 +958,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 
 	if o.translatedLink {
 		if err == nil {
-			// Remove any current symlink or file, if one exsits
+			// Remove any current symlink or file, if one exists
 			if _, err := os.Lstat(o.path); err == nil {
 				if removeErr := os.Remove(o.path); removeErr != nil {
 					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)

backend/local/read_device_unix.go

@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
 		fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
 		return devUnset
 	}
-	return uint64(statT.Dev)
+	return uint64(statT.Dev) // nolint: unconvert
 }

backend/mega/mega.go

@@ -98,7 +98,7 @@ type Fs struct {
 	opt        Options      // parsed config options
 	features   *fs.Features // optional features
 	srv        *mega.Mega   // the connection to the server
-	pacer      *pacer.Pacer // pacer for API calls
+	pacer      *fs.Pacer    // pacer for API calls
 	rootNodeMu sync.Mutex   // mutex for _rootNode
 	_rootNode  *mega.Node   // root node - call findRoot to use this
 	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir
@@ -217,7 +217,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   srv,
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
@@ -402,6 +402,27 @@ func (f *Fs) clearRoot() {
 	//log.Printf("cleared root directory")
 }
 
+// CleanUp deletes all files currently in trash
+func (f *Fs) CleanUp() (err error) {
+	trash := f.srv.FS.GetTrash()
+	items := []*mega.Node{}
+	_, err = f.list(trash, func(item *mega.Node) bool {
+		items = append(items, item)
+		return false
+	})
+	if err != nil {
+		return errors.Wrap(err, "CleanUp failed to list items in trash")
+	}
+	// similar to f.deleteNode(trash) but with HardDelete as true
+	for _, item := range items {
+		err = f.pacer.Call(func() (bool, error) {
+			err = f.srv.Delete(item, true)
+			return shouldRetry(err)
+		})
+	}
+	return err
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -497,7 +518,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, obect, leaf and error
+// Returns the dirNode, object, leaf and error
 //
 // Used to create new objects
 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -523,10 +544,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return exisitingObj, exisitingObj.Update(in, src, options...)
+		return existingObj, existingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -847,14 +868,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.deleteNode(srcDirNode)
 		if err != nil {
-			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1076,6 +1097,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
+	if size < 0 {
+		return errors.New("mega backend can't upload a file of unknown length")
+	}
 	//modTime := src.ModTime()
 	remote := o.Remote()
 
@@ -1126,7 +1150,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return errors.Wrap(err, "failed to finish upload")
 	}
 
-	// If the upload succeded and the original object existed, then delete it
+	// If the upload succeeded and the original object existed, then delete it
 	if o.info != nil {
 		err = o.fs.deleteNode(o.info)
 		if err != nil {

backend/onedrive/api/types.go

@@ -25,7 +25,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	out := e.ErrorInfo.Code
 	if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +35,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // Identity represents an identity of an actor. For example, and actor
@@ -295,9 +295,9 @@ func (i *Item) GetID() string {
 	return i.ID
 }
 
-// GetDriveID returns a normalized ParentReferance of the item
+// GetDriveID returns a normalized ParentReference of the item
 func (i *Item) GetDriveID() string {
-	return i.GetParentReferance().DriveID
+	return i.GetParentReference().DriveID
 }
 
 // GetName returns a normalized Name of the item
@@ -398,8 +398,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
 	return i.LastModifiedDateTime
 }
 
-// GetParentReferance returns a normalized ParentReferance of the item
-func (i *Item) GetParentReferance() *ItemReference {
+// GetParentReference returns a normalized ParentReference of the item
+func (i *Item) GetParentReference() *ItemReference {
 	if i.IsRemote() && i.ParentReference == nil {
 		return i.RemoteItem.ParentReference
 	}

backend/onedrive/onedrive.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/lib/atexit"
+
 	"github.com/ncw/rclone/backend/onedrive/api"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -227,7 +229,7 @@ that the chunks will be buffered into memory.`,
 			Advanced: true,
 		}, {
 			Name:     "drive_type",
-			Help:     "The type of the drive ( personal | business | documentLibrary )",
+			Help:     "The type of the drive ( " + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + " )",
 			Default:  "",
 			Advanced: true,
 		}, {
@@ -261,7 +263,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the one drive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
@@ -324,19 +326,24 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRety := false
+	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRety = true
+		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
 // if `relPath` == "", it reads the metadata for the item with that ID.
+//
+// We address items using the pattern `drives/driveID/items/itemID:/relativePath`
+// instead of simply using `drives/driveID/root:/itemPath` because it works for
+// "shared with me" folders in OneDrive Personal (See #2536, #2778)
+// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
+	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -475,7 +482,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		driveID:   opt.DriveID,
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
-		pacer:     pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -703,9 +710,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			id := info.GetID()
 			f.dirCache.Put(remote, id)
 			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-			if folder != nil {
-				d.SetItems(folder.ChildCount)
-			}
+			d.SetItems(folder.ChildCount)
 			entries = append(entries, d)
 		} else {
 			o, err := f.newObjectWithInfo(remote, info)
@@ -819,9 +824,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -1340,12 +1342,12 @@ func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
 		}
 	} else {
 		opts = rest.Opts{
 			Method: "PATCH",
-			Path:   "/root:/" + rest.URLPathEscape(o.srvPath()),
+			Path:   "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
 		}
 	}
 	update := api.SetFileSystemInfo{
@@ -1488,26 +1490,43 @@ func (o *Object) cancelUploadSession(url string) (err error) {
 // uploadMultipart uploads a file using multipart upload
 func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size <= 0 {
-		panic("size passed into uploadMultipart must be > 0")
+		return nil, errors.New("unknown-sized upload not supported")
 	}
 
+	uploadURLChan := make(chan string, 1)
+	gracefulCancel := func() {
+		uploadURL, ok := <-uploadURLChan
+		// Reading from uploadURLChan blocks the atexit process until
+		// we are able to use uploadURL to cancel the upload
+		if !ok { // createUploadSession failed - no need to cancel upload
+			return
+		}
+
+		fs.Debugf(o, "Cancelling multipart upload")
+		cancelErr := o.cancelUploadSession(uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+		}
+	}
+	cancelFuncHandle := atexit.Register(gracefulCancel)
+
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(modTime)
 	if err != nil {
+		close(uploadURLChan)
+		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
+	uploadURLChan <- uploadURL
 
-	// Cancel the session if something went wrong
 	defer func() {
 		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.cancelUploadSession(uploadURL)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+			fs.Debugf(o, "Error encountered during upload: %v", err)
+			gracefulCancel()
 		}
+		atexit.Unregister(cancelFuncHandle)
 	}()
 
 	// Upload the chunks
@@ -1535,7 +1554,7 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
 // This function will set modtime after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
-		panic("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
+		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -1602,7 +1621,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	} else if size == 0 {
 		info, err = o.uploadSinglepart(in, size, modTime)
 	} else {
-		panic("src file size must be >= 0")
+		return errors.New("unknown-sized upload not supported")
 	}
 	if err != nil {
 		return err
@@ -1668,6 +1687,21 @@ func getRelativePathInsideBase(base, target string) (string, bool) {
 	return "", false
 }
 
+// Adds a ":" at the end of `remotePath` in a proper manner.
+// If `remotePath` already ends with "/", change it to ":/"
+// If `remotePath` is "", return "".
+// A workaround for #2720 and #3039
+func withTrailingColon(remotePath string) string {
+	if remotePath == "" {
+		return ""
+	}
+
+	if strings.HasSuffix(remotePath, "/") {
+		return remotePath[:len(remotePath)-1] + ":/"
+	}
+	return remotePath + ":"
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -65,7 +65,7 @@ type Fs struct {
 	opt      Options            // parsed options
 	features *fs.Features       // optional features
 	srv      *rest.Client       // the connection to the server
-	pacer    *pacer.Pacer       // To pace and retry the API calls
+	pacer    *fs.Pacer          // To pace and retry the API calls
 	session  UserSessionInfo    // contains the session data
 	dirCache *dircache.DirCache // Map of directory path to directory id
 }
@@ -119,7 +119,7 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -144,7 +144,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 
 	f.dirCache = dircache.New(root, "0", f)
@@ -287,9 +287,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -785,7 +782,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, folder.FolderID)
-		d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
+		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
 		entries = append(entries, d)
 	}

backend/opendrive/types.go

@@ -13,7 +13,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }

backend/pcloud/api/types.go

@@ -41,7 +41,7 @@ type Error struct {
 	ErrorString string `json:"error"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
 }
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
 	return e
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // Item describes a folder or a file as returned by Get Folder Items and others
@@ -161,7 +161,6 @@ type UserInfo struct {
 	PublicLinkQuota       int64  `json:"publiclinkquota"`
 	Email                 string `json:"email"`
 	UserID                int    `json:"userid"`
-	Result                int    `json:"result"`
 	Quota                 int64  `json:"quota"`
 	TrashRevretentionDays int    `json:"trashrevretentiondays"`
 	Premium               bool   `json:"premium"`

backend/pcloud/pcloud.go

@@ -95,7 +95,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
@@ -254,7 +254,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         false,
@@ -385,7 +385,7 @@ func fileIDtoNumber(fileID string) string {
 	if len(fileID) > 0 && fileID[0] == 'f' {
 		return fileID[1:]
 	}
-	fs.Debugf(nil, "Invalid filee id %q", fileID)
+	fs.Debugf(nil, "Invalid file id %q", fileID)
 	return fileID
 }
 

backend/qingstor/qingstor.go

@@ -449,7 +449,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	_, err = bucketInit.PutObject(key, &req)
 	if err != nil {
-		fs.Debugf(f, "Copied Faild, API Error: %v", err)
+		fs.Debugf(f, "Copy Failed, API Error: %v", err)
 		return nil, err
 	}
 	return f.NewObject(remote)
@@ -756,7 +756,7 @@ func (f *Fs) Mkdir(dir string) error {
 		}
 		switch *statistics.Status {
 		case "deleted":
-			fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
+			fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
 			time.Sleep(time.Second * 1)
 			retries++
 			continue
@@ -875,7 +875,7 @@ func (o *Object) readMetaData() (err error) {
 	fs.Debugf(o, "Read metadata of key: %s", key)
 	resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
 	if err != nil {
-		fs.Debugf(o, "Read metadata faild, API Error: %v", err)
+		fs.Debugf(o, "Read metadata failed, API Error: %v", err)
 		if e, ok := err.(*qsErr.QingStorError); ok {
 			if e.StatusCode == http.StatusNotFound {
 				return fs.ErrorObjectNotFound

backend/qingstor/upload.go

@@ -143,7 +143,7 @@ func (u *uploader) init() {
 
 		// Try to adjust partSize if it is too small and account for
 		// integer division truncation.
-		if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
+		if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
 			// Add one to the part size to account for remainders
 			// during the size calculation. e.g odd number of bytes.
 			u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -163,7 +163,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 
 	_, err := bucketInit.PutObject(u.cfg.key, &req)
 	if err == nil {
-		fs.Debugf(u, "Upload single objcet finished")
+		fs.Debugf(u, "Upload single object finished")
 	}
 	return err
 }

backend/s3/s3.go

@@ -131,6 +131,9 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
+			}, {
+				Value: "eu-north-1",
+				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -234,10 +237,10 @@ func init() {
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.softlayer.net",
-				Help:  "Great Britan Endpoint",
+				Help:  "Great Britain Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
-				Help:  "Great Britan Private Endpoint",
+				Help:  "Great Britain Private Endpoint",
 			}, {
 				Value: "s3.ap-geo.objectstorage.softlayer.net",
 				Help:  "APAC Cross Regional Endpoint",
@@ -343,7 +346,7 @@ func init() {
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
 			Provider: "!AWS,IBMCOS,Alibaba",
 			Examples: []fs.OptionExample{{
-				Value:    "objects-us-west-1.dream.io",
+				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
 			}, {
@@ -392,6 +395,9 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
+			}, {
+				Value: "eu-north-1",
+				Help:  "EU (Stockholm) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -444,7 +450,7 @@ func init() {
 				Help:  "US East Region Flex",
 			}, {
 				Value: "us-south-standard",
-				Help:  "US Sout hRegion Standard",
+				Help:  "US South Region Standard",
 			}, {
 				Value: "us-south-vault",
 				Help:  "US South Region Vault",
@@ -468,16 +474,16 @@ func init() {
 				Help:  "EU Cross Region Flex",
 			}, {
 				Value: "eu-gb-standard",
-				Help:  "Great Britan Standard",
+				Help:  "Great Britain Standard",
 			}, {
 				Value: "eu-gb-vault",
-				Help:  "Great Britan Vault",
+				Help:  "Great Britain Vault",
 			}, {
 				Value: "eu-gb-cold",
-				Help:  "Great Britan Cold",
+				Help:  "Great Britain Cold",
 			}, {
 				Value: "eu-gb-flex",
-				Help:  "Great Britan Flex",
+				Help:  "Great Britain Flex",
 			}, {
 				Value: "ap-standard",
 				Help:  "APAC Standard",
@@ -639,6 +645,9 @@ isn't set then "acl" is used instead.`,
 			}, {
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
+			}, {
+				Value: "DEEP_ARCHIVE",
+				Help:  "Glacier Deep Archive storage class",
 			}},
 		}, {
 			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
@@ -723,6 +732,14 @@ If it is set then rclone will use v2 authentication.
 Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "use_accelerate_endpoint",
+			Provider: "AWS",
+			Help: `If true use the AWS S3 accelerated endpoint.
+
+See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
+			Default:  false,
+			Advanced: true,
 		}},
 	})
 }
@@ -743,25 +760,26 @@ const (
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider             string        `config:"provider"`
-	EnvAuth              bool          `config:"env_auth"`
-	AccessKeyID          string        `config:"access_key_id"`
-	SecretAccessKey      string        `config:"secret_access_key"`
-	Region               string        `config:"region"`
-	Endpoint             string        `config:"endpoint"`
-	LocationConstraint   string        `config:"location_constraint"`
-	ACL                  string        `config:"acl"`
-	BucketACL            string        `config:"bucket_acl"`
-	ServerSideEncryption string        `config:"server_side_encryption"`
-	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
-	StorageClass         string        `config:"storage_class"`
-	UploadCutoff         fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum      bool          `config:"disable_checksum"`
-	SessionToken         string        `config:"session_token"`
-	UploadConcurrency    int           `config:"upload_concurrency"`
-	ForcePathStyle       bool          `config:"force_path_style"`
-	V2Auth               bool          `config:"v2_auth"`
+	Provider              string        `config:"provider"`
+	EnvAuth               bool          `config:"env_auth"`
+	AccessKeyID           string        `config:"access_key_id"`
+	SecretAccessKey       string        `config:"secret_access_key"`
+	Region                string        `config:"region"`
+	Endpoint              string        `config:"endpoint"`
+	LocationConstraint    string        `config:"location_constraint"`
+	ACL                   string        `config:"acl"`
+	BucketACL             string        `config:"bucket_acl"`
+	ServerSideEncryption  string        `config:"server_side_encryption"`
+	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
+	StorageClass          string        `config:"storage_class"`
+	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
+	DisableChecksum       bool          `config:"disable_checksum"`
+	SessionToken          string        `config:"session_token"`
+	UploadConcurrency     int           `config:"upload_concurrency"`
+	ForcePathStyle        bool          `config:"force_path_style"`
+	V2Auth                bool          `config:"v2_auth"`
+	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
 }
 
 // Fs represents a remote s3 server
@@ -776,7 +794,7 @@ type Fs struct {
 	bucketOKMu    sync.Mutex       // mutex to protect bucket OK
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
-	pacer         *pacer.Pacer     // To pace the API calls
+	pacer         *fs.Pacer        // To pace the API calls
 	srv           *http.Client     // a plain http client
 }
 
@@ -836,7 +854,7 @@ var retryErrorCodes = []int{
 func (f *Fs) shouldRetry(err error) (bool, error) {
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
-		// Simple case, check the original embedded error in case it's generically retriable
+		// Simple case, check the original embedded error in case it's generically retryable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
@@ -935,14 +953,15 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
-		WithS3ForcePathStyle(opt.ForcePathStyle)
+		WithS3ForcePathStyle(opt.ForcePathStyle).
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
@@ -1049,7 +1068,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		c:      c,
 		bucket: bucket,
 		ses:    ses,
-		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
+		pacer:  fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -14,6 +14,7 @@ import (
 	"os/user"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -427,6 +428,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
+	return NewFsWithConnection(name, root, opt, sshConfig)
+}
+
+// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
+// the host specified in the ssh.ClientConfig
+func NewFsWithConnection(name string, root string, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
 		name:      name,
 		root:      root,
@@ -807,6 +814,47 @@ func (f *Fs) Hashes() hash.Set {
 	return set
 }
 
+// About gets usage stats
+func (f *Fs) About() (*fs.Usage, error) {
+	c, err := f.getSftpConnection()
+	if err != nil {
+		return nil, errors.Wrap(err, "About get SFTP connection")
+	}
+	session, err := c.sshClient.NewSession()
+	f.putSftpConnection(&c, err)
+	if err != nil {
+		return nil, errors.Wrap(err, "About put SFTP connection")
+	}
+
+	var stdout, stderr bytes.Buffer
+	session.Stdout = &stdout
+	session.Stderr = &stderr
+	escapedPath := shellEscape(f.root)
+	if f.opt.PathOverride != "" {
+		escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
+	}
+	if len(escapedPath) == 0 {
+		escapedPath = "/"
+	}
+	err = session.Run("df -k " + escapedPath)
+	if err != nil {
+		_ = session.Close()
+		return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.")
+	}
+	_ = session.Close()
+
+	usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes())
+	if usageTotal < 0 || usageUsed < 0 || usageAvail < 0 {
+		return nil, errors.Wrap(err, "About failed to parse information")
+	}
+	usage := &fs.Usage{
+		Total: fs.NewUsageValue(usageTotal),
+		Used:  fs.NewUsageValue(usageUsed),
+		Free:  fs.NewUsageValue(usageAvail),
+	}
+	return usage, nil
+}
+
 // Fs is the filesystem this remote sftp file object is located within
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -897,6 +945,34 @@ func parseHash(bytes []byte) string {
 	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
 }
 
+// Parses the byte array output from the SSH session
+// returned by an invocation of df into
+// the disk size, used space, and avaliable space on the disk, in that order.
+// Only works when `df` has output info on only one disk
+func parseUsage(bytes []byte) (int64, int64, int64) {
+	lines := strings.Split(string(bytes), "\n")
+	if len(lines) < 2 {
+		return -1, -1, -1
+	}
+	split := strings.Fields(lines[1])
+	if len(split) < 6 {
+		return -1, -1, -1
+	}
+	spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
+	if err != nil {
+		return -1, -1, -1
+	}
+	spaceUsed, err := strconv.ParseInt(split[2], 10, 64)
+	if err != nil {
+		return -1, -1, -1
+	}
+	spaceAvail, err := strconv.ParseInt(split[3], 10, 64)
+	if err != nil {
+		return -1, -1, -1
+	}
+	return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
+}
+
 // Size returns the size in bytes of the remote sftp file
 func (o *Object) Size() int64 {
 	return o.size

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -35,3 +35,17 @@ func TestParseHash(t *testing.T) {
 		assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
 	}
 }
+
+func TestParseUsage(t *testing.T) {
+	for i, test := range []struct {
+		sshOutput string
+		usage     [3]int64
+	}{
+		{"Filesystem     1K-blocks     Used Available Use% Mounted on\n/dev/root       91283092 81111888  10154820  89% /", [3]int64{93473886208, 83058573312, 10398535680}},
+		{"Filesystem     1K-blocks  Used Available Use% Mounted on\ntmpfs             818256  1636    816620   1% /run", [3]int64{837894144, 1675264, 836218880}},
+		{"Filesystem   1024-blocks     Used Available Capacity iused      ifree %iused  Mounted on\n/dev/disk0s2   244277768 94454848 149566920    39%  997820 4293969459    0%   /", [3]int64{250140434432, 96721764352, 153156526080}},
+	} {
+		gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput))
+		assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
+	}
+}

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.9
+// +build plan9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/sftp/stringlock_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/swift/auth.go

@@ -2,6 +2,7 @@ package swift
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/ncw/swift"
 )
@@ -65,6 +66,14 @@ func (a *auth) Token() string {
 	return a.parentAuth.Token()
 }
 
+// Expires returns the time the token expires if known or Zero if not.
+func (a *auth) Expires() (t time.Time) {
+	if do, ok := a.parentAuth.(swift.Expireser); ok {
+		t = do.Expires()
+	}
+	return t
+}
+
 // The CDN url if available
 func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
@@ -74,4 +83,7 @@ func (a *auth) CdnUrl() string { // nolint
 }
 
 // Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)
+var (
+	_ swift.Authenticator = (*auth)(nil)
+	_ swift.Expireser     = (*auth)(nil)
+)

backend/swift/swift.go

@@ -195,7 +195,7 @@ type Options struct {
 	StorageURL                  string        `config:"storage_url"`
 	AuthToken                   string        `config:"auth_token"`
 	AuthVersion                 int           `config:"auth_version"`
-	ApplicationCredentialId     string        `config:"application_credential_id"`
+	ApplicationCredentialID     string        `config:"application_credential_id"`
 	ApplicationCredentialName   string        `config:"application_credential_name"`
 	ApplicationCredentialSecret string        `config:"application_credential_secret"`
 	StoragePolicy               string        `config:"storage_policy"`
@@ -216,7 +216,7 @@ type Fs struct {
 	containerOK       bool              // true if we have created the container
 	segmentsContainer string            // container to store the segments (if any) in
 	noCheckContainer  bool              // don't check the container before creating it
-	pacer             *pacer.Pacer      // To pace the API calls
+	pacer             *fs.Pacer         // To pace the API calls
 }
 
 // Object describes a swift object
@@ -286,6 +286,31 @@ func shouldRetry(err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
+// shouldRetryHeaders returns a boolean as to whether this err
+// deserves to be retried.  It reads the headers passed in looking for
+// `Retry-After`. It returns the err as a convenience
+func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
+	if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
+		if value := headers["Retry-After"]; value != "" {
+			retryAfter, parseErr := strconv.Atoi(value)
+			if parseErr != nil {
+				fs.Errorf(nil, "Failed to parse Retry-After: %q: %v", value, parseErr)
+			} else {
+				duration := time.Second * time.Duration(retryAfter)
+				if duration <= 60*time.Second {
+					// Do a short sleep immediately
+					fs.Debugf(nil, "Sleeping for %v to obey Retry-After", duration)
+					time.Sleep(duration)
+					return true, err
+				}
+				// Delay a long sleep for a retry
+				return false, fserrors.NewErrorRetryAfter(duration)
+			}
+		}
+	}
+	return shouldRetry(err)
+}
+
 // Pattern to match a swift path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -317,7 +342,7 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 		StorageUrl:                  opt.StorageURL,
 		AuthToken:                   opt.AuthToken,
 		AuthVersion:                 opt.AuthVersion,
-		ApplicationCredentialId:     opt.ApplicationCredentialId,
+		ApplicationCredentialId:     opt.ApplicationCredentialID,
 		ApplicationCredentialName:   opt.ApplicationCredentialName,
 		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
 		EndpointType:                swift.EndpointType(opt.EndpointType),
@@ -401,7 +426,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		segmentsContainer: container + "_segments",
 		root:              directory,
 		noCheckContainer:  noCheckContainer,
-		pacer:             pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
+		pacer:             fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -413,8 +438,9 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		// Check to see if the object exists - ignoring directory markers
 		var info swift.Object
 		err = f.pacer.Call(func() (bool, error) {
-			info, _, err = f.c.Object(container, directory)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			info, rxHeaders, err = f.c.Object(container, directory)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			f.root = path.Dir(directory)
@@ -430,7 +456,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 	return f, nil
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -723,8 +749,9 @@ func (f *Fs) Mkdir(dir string) error {
 	var err error = swift.ContainerNotFound
 	if !f.noCheckContainer {
 		err = f.pacer.Call(func() (bool, error) {
-			_, _, err = f.c.Container(f.container)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			_, rxHeaders, err = f.c.Container(f.container)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 	}
 	if err == swift.ContainerNotFound {
@@ -816,8 +843,9 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	srcFs := srcObj.fs
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err != nil {
 		return nil, err
@@ -927,7 +955,7 @@ func (o *Object) readMetaData() (err error) {
 	var h swift.Headers
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, h, err = o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
-		return shouldRetry(err)
+		return shouldRetryHeaders(h, err)
 	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
@@ -1002,8 +1030,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	headers := fs.OpenOptionHeaders(options)
 	_, isRanging := headers["Range"]
 	err = o.fs.pacer.Call(func() (bool, error) {
-		in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		in, rxHeaders, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return
 }
@@ -1074,8 +1103,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	// Create the segmentsContainer if it doesn't exist
 	var err error
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		_, rxHeaders, err = o.fs.c.Container(o.fs.segmentsContainer)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
@@ -1115,8 +1145,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 		segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
 		fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			rxHeaders, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return "", err
@@ -1129,8 +1160,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	emptyReader := bytes.NewReader(nil)
 	manifestName := o.fs.root + o.remote
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return uniquePrefix + "/", err
 }
@@ -1174,7 +1206,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		var rxHeaders swift.Headers
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
-			return shouldRetry(err)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return err

backend/swift/swift_internal_test.go

@@ -1,6 +1,13 @@
 package swift
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/swift"
+	"github.com/stretchr/testify/assert"
+)
 
 func TestInternalUrlEncode(t *testing.T) {
 	for _, test := range []struct {
@@ -23,3 +30,37 @@ func TestInternalUrlEncode(t *testing.T) {
 		}
 	}
 }
+
+func TestInternalShouldRetryHeaders(t *testing.T) {
+	headers := swift.Headers{
+		"Content-Length": "64",
+		"Content-Type":   "text/html; charset=UTF-8",
+		"Date":           "Mon: 18 Mar 2019 12:11:23 GMT",
+		"Retry-After":    "1",
+	}
+	err := &swift.Error{
+		StatusCode: 429,
+		Text:       "Too Many Requests",
+	}
+
+	// Short sleep should just do the sleep
+	start := time.Now()
+	retry, gotErr := shouldRetryHeaders(headers, err)
+	dt := time.Since(start)
+	assert.True(t, retry)
+	assert.Equal(t, err, gotErr)
+	assert.True(t, dt > time.Second/2)
+
+	// Long sleep should return RetryError
+	headers["Retry-After"] = "3600"
+	start = time.Now()
+	retry, gotErr = shouldRetryHeaders(headers, err)
+	dt = time.Since(start)
+	assert.True(t, dt < time.Second)
+	assert.False(t, retry)
+	assert.Equal(t, true, fserrors.IsRetryAfterError(gotErr))
+	after := gotErr.(fserrors.RetryAfter).RetryAfter()
+	dt = after.Sub(start)
+	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
+
+}

backend/union/union.go

@@ -177,8 +177,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // At least one value will be written to the channel,
 // specifying the initial value and updated values might
 // follow. A 0 Duration should pause the polling.
-// The ChangeNotify implemantion must empty the channel
-// regulary. When the channel gets closed, the implemantion
+// The ChangeNotify implementation must empty the channel
+// regularly. When the channel gets closed, the implementation
 // should stop polling and release resources.
 func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
 	var remoteChans []chan time.Duration

backend/webdav/api/types.go

@@ -66,12 +66,13 @@ type Response struct {
 // Note that status collects all the status values for which we just
 // check the first is OK.
 type Prop struct {
-	Status    []string  `xml:"DAV: status"`
-	Name      string    `xml:"DAV: prop>displayname,omitempty"`
-	Type      *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
-	Size      int64     `xml:"DAV: prop>getcontentlength,omitempty"`
-	Modified  Time      `xml:"DAV: prop>getlastmodified,omitempty"`
-	Checksums []string  `xml:"prop>checksums>checksum,omitempty"`
+	Status       []string  `xml:"DAV: status"`
+	Name         string    `xml:"DAV: prop>displayname,omitempty"`
+	Type         *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	IsCollection *string   `xml:"DAV: prop>iscollection,omitempty"` // this is a Microsoft extension see #2716
+	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -123,7 +124,7 @@ type PropValue struct {
 	Value   string   `xml:",chardata"`
 }
 
-// Error is used to desribe webdav errors
+// Error is used to describe webdav errors
 //
 // <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
 //   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
@@ -136,7 +137,7 @@ type Error struct {
 	StatusCode int
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	var out []string
 	if e.Message != "" {

backend/webdav/odrvcookie/fetch.go

@@ -102,7 +102,7 @@ func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
 func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
 	spRoot, err := url.Parse(ca.endpoint)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
+		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
 	}
 
 	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
@@ -121,7 +121,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 		Jar: jar,
 	}
 
-	// Send the previously aquired Token as a Post parameter
+	// Send the previously acquired Token as a Post parameter
 	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
 		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
 	}

backend/webdav/odrvcookie/renew.go

@@ -2,13 +2,10 @@ package odrvcookie
 
 import (
 	"time"
-
-	"github.com/ncw/rclone/lib/rest"
 )
 
 // CookieRenew holds information for the renew
 type CookieRenew struct {
-	srv     *rest.Client
 	timer   *time.Ticker
 	renewFn func()
 }

backend/webdav/webdav.go

@@ -101,7 +101,7 @@ type Fs struct {
 	endpoint           *url.URL      // URL of the host
 	endpointURL        string        // endpoint as a string
 	srv                *rest.Client  // the connection to the one drive server
-	pacer              *pacer.Pacer  // pacer for API calls
+	pacer              *fs.Pacer     // pacer for API calls
 	precision          time.Duration // mod time precision
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
@@ -172,6 +172,18 @@ func itemIsDir(item *api.Response) bool {
 		}
 		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
 	}
+	// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
+	// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
+	if t := item.Props.IsCollection; t != nil {
+		switch x := strings.ToLower(*t); x {
+		case "0", "false":
+			return false
+		case "1", "true":
+			return true
+		default:
+			fs.Debugf(nil, "Unknown value %q for IsCollection", x)
+		}
+	}
 	return false
 }
 
@@ -244,7 +256,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// addShlash makes sure s is terminated with a / if non empty
+// addSlash makes sure s is terminated with a / if non empty
 func addSlash(s string) string {
 	if s != "" && !strings.HasSuffix(s, "/") {
 		s += "/"
@@ -306,7 +318,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		endpoint:    u,
 		endpointURL: u.String(),
 		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}
 	f.features = (&fs.Features{
@@ -632,10 +644,18 @@ func (f *Fs) _mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	return f.pacer.Call(func() (bool, error) {
+	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// already exists
+		// owncloud returns 423/StatusLocked if the create is already in progress
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
+			return nil
+		}
+	}
+	return err
 }
 
 // mkdir makes the directory and parents using native paths
@@ -643,11 +663,7 @@ func (f *Fs) mkdir(dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
-		// already exists
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
-			return nil
-		}
-		// parent does not exist
+		// parent does not exist so create it first then try again
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
@@ -900,11 +916,13 @@ func (f *Fs) About() (*fs.Usage, error) {
 		return nil, errors.Wrap(err, "about call failed")
 	}
 	usage := &fs.Usage{}
-	if q.Available >= 0 && q.Used >= 0 {
-		usage.Total = fs.NewUsageValue(q.Available + q.Used)
-	}
-	if q.Used >= 0 {
-		usage.Used = fs.NewUsageValue(q.Used)
+	if q.Available != 0 || q.Used != 0 {
+		if q.Available >= 0 && q.Used >= 0 {
+			usage.Total = fs.NewUsageValue(q.Available + q.Used)
+		}
+		if q.Used >= 0 {
+			usage.Used = fs.NewUsageValue(q.Used)
+		}
 	}
 	return usage, nil
 }

backend/yandex/api/types.go

@@ -56,7 +56,7 @@ type AsyncInfo struct {
 	Templated bool   `json:"templated"`
 }
 
-// AsyncStatus is returned when requesting the status of an async operations. Possble values in-progress, success, failure
+// AsyncStatus is returned when requesting the status of an async operations. Possible values in-progress, success, failure
 type AsyncStatus struct {
 	Status string `json:"status"`
 }

backend/yandex/yandex.go

@@ -93,7 +93,7 @@ type Fs struct {
 	opt      Options      // parsed options
 	features *fs.Features // optional features
 	srv      *rest.Client // the connection to the yandex server
-	pacer    *pacer.Pacer // pacer for API calls
+	pacer    *fs.Pacer    // pacer for API calls
 	diskRoot string       // root path with "disk:/" container name
 }
 
@@ -269,7 +269,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -307,7 +307,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs
 		if err != nil {
 			return nil, errors.Wrap(err, "error parsing time in directory item")
 		}
-		d := fs.NewDir(remote, t).SetSize(int64(object.Size))
+		d := fs.NewDir(remote, t).SetSize(object.Size)
 		return d, nil
 	case "file":
 		o, err := f.newObjectWithInfo(remote, object)
@@ -634,7 +634,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copys or moves directories or files depending on the mthod parameter
+// copyOrMoves copies or moves directories or files depending on the method parameter
 func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -1107,7 +1107,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	//if file uploaded sucessfully then return metadata
+	//if file uploaded successfully then return metadata
 	o.modTime = modTime
 	o.md5sum = ""                   // according to unit tests after put the md5 is empty.
 	o.size = int64(in1.BytesRead()) // better solution o.readMetaData() ?

bin/get-github-release.go

@@ -8,6 +8,8 @@
 package main
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -15,13 +17,18 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/lib/rest"
+	"golang.org/x/net/html"
 	"golang.org/x/sys/unix"
 )
 
@@ -30,8 +37,15 @@ var (
 	install = flag.Bool("install", false, "Install the downloaded package using sudo dpkg -i.")
 	extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.")
 	bindir  = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.")
+	useAPI  = flag.Bool("use-api", false, "Use the API for finding the release instead of scraping the page.")
 	// Globals
-	matchProject = regexp.MustCompile(`^(\w+)/(\w+)$`)
+	matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
+	osAliases    = map[string][]string{
+		"darwin": []string{"macos", "osx"},
+	}
+	archAliases = map[string][]string{
+		"amd64": []string{"x86_64"},
+	}
 )
 
 // A github release
@@ -113,25 +127,41 @@ func writable(path string) bool {
 
 // Directory to install releases in by default
 //
-// Find writable directories on $PATH.  Use the first writable
-// directory which is in $HOME or failing that the first writable
-// directory.
+// Find writable directories on $PATH.  Use $GOPATH/bin if that is on
+// the path and writable or use the first writable directory which is
+// in $HOME or failing that the first writable directory.
 //
 // Returns "" if none of the above were found
 func defaultBinDir() string {
 	home := os.Getenv("HOME")
-	var binDir string
+	var (
+		bin       string
+		homeBin   string
+		goHomeBin string
+		gopath    = os.Getenv("GOPATH")
+	)
 	for _, dir := range strings.Split(os.Getenv("PATH"), ":") {
 		if writable(dir) {
 			if strings.HasPrefix(dir, home) {
-				return dir
+				if homeBin != "" {
+					homeBin = dir
+				}
+				if gopath != "" && strings.HasPrefix(dir, gopath) && goHomeBin == "" {
+					goHomeBin = dir
+				}
 			}
-			if binDir != "" {
-				binDir = dir
+			if bin == "" {
+				bin = dir
 			}
 		}
 	}
-	return binDir
+	if goHomeBin != "" {
+		return goHomeBin
+	}
+	if homeBin != "" {
+		return homeBin
+	}
+	return bin
 }
 
 // read the body or an error message
@@ -175,7 +205,8 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	}
 
 	for _, asset := range release.Assets {
-		if matchName.MatchString(asset.Name) {
+		//log.Printf("Finding %s", asset.Name)
+		if matchName.MatchString(asset.Name) && isOurOsArch(asset.Name) {
 			return asset.BrowserDownloadURL, asset.Name
 		}
 	}
@@ -183,6 +214,73 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	return "", ""
 }
 
+// Get an asset URL and name by scraping the downloads page
+//
+// This doesn't use the API so isn't rate limited when not using GITHUB login details
+func getAssetFromReleasesPage(project string, matchName *regexp.Regexp) (assetURL string, assetName string) {
+	baseURL := "https://github.com/" + project + "/releases"
+	log.Printf("Fetching asset info for %q from %q", project, baseURL)
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		log.Fatalf("URL Parse failed: %v", err)
+	}
+	resp, err := http.Get(baseURL)
+	if err != nil {
+		log.Fatalf("Failed to fetch release info %q: %v", baseURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("Error: %s", readBody(resp.Body))
+		log.Fatalf("Bad status %d when fetching %q release info: %s", resp.StatusCode, baseURL, resp.Status)
+	}
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		log.Fatalf("Failed to parse web page: %v", err)
+	}
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			for _, a := range n.Attr {
+				if a.Key == "href" {
+					if name := path.Base(a.Val); matchName.MatchString(name) && isOurOsArch(name) {
+						if u, err := rest.URLJoin(base, a.Val); err == nil {
+							if assetName == "" {
+								assetName = name
+								assetURL = u.String()
+							}
+						}
+					}
+					break
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	if assetName == "" || assetURL == "" {
+		log.Fatalf("Didn't find URL in page")
+	}
+	return assetURL, assetName
+}
+
+// isOurOsArch returns true if s contains our OS and our Arch
+func isOurOsArch(s string) bool {
+	s = strings.ToLower(s)
+	check := func(base string, aliases map[string][]string) bool {
+		names := []string{base}
+		names = append(names, aliases[base]...)
+		for _, name := range names {
+			if strings.Contains(s, name) {
+				return true
+			}
+		}
+		return false
+	}
+	return check(runtime.GOARCH, archAliases) && check(runtime.GOOS, osAliases)
+}
+
 // get a file for download
 func getFile(url, fileName string) {
 	log.Printf("Downloading %q from %q", fileName, url)
@@ -229,6 +327,66 @@ func run(args ...string) {
 	}
 }
 
+// Untars fileName from srcFile
+func untar(srcFile, fileName, extractDir string) {
+	f, err := os.Open(srcFile)
+	if err != nil {
+		log.Fatalf("Couldn't open tar: %v", err)
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			log.Fatalf("Couldn't close tar: %v", err)
+		}
+	}()
+
+	var in io.Reader = f
+
+	srcExt := filepath.Ext(srcFile)
+	if srcExt == ".gz" || srcExt == ".tgz" {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			log.Fatalf("Couldn't open gzip: %v", err)
+		}
+		in = gzf
+	}
+
+	tarReader := tar.NewReader(in)
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("Trouble reading tar file: %v", err)
+		}
+		name := header.Name
+		switch header.Typeflag {
+		case tar.TypeReg:
+			baseName := filepath.Base(name)
+			if baseName == fileName {
+				outPath := filepath.Join(extractDir, fileName)
+				out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+				if err != nil {
+					log.Fatalf("Couldn't open output file: %v", err)
+				}
+				defer func() {
+					err := out.Close()
+					if err != nil {
+						log.Fatalf("Couldn't close output: %v", err)
+					}
+				}()
+				n, err := io.Copy(out, tarReader)
+				if err != nil {
+					log.Fatalf("Couldn't write output file: %v", err)
+				}
+				log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
+			}
+		}
+	}
+}
+
 func main() {
 	flag.Parse()
 	args := flag.Args()
@@ -244,7 +402,12 @@ func main() {
 		log.Fatalf("Invalid regexp for name %q: %v", nameRe, err)
 	}
 
-	assetURL, assetName := getAsset(project, matchName)
+	var assetURL, assetName string
+	if *useAPI {
+		assetURL, assetName = getAsset(project, matchName)
+	} else {
+		assetURL, assetName = getAssetFromReleasesPage(project, matchName)
+	}
 	fileName := filepath.Join(os.TempDir(), assetName)
 	getFile(assetURL, fileName)
 
@@ -257,8 +420,6 @@ func main() {
 			log.Fatalf("Need to set -bindir")
 		}
 		log.Printf("Unpacking %s from %s and installing into %s", *extract, fileName, *bindir)
-		run("tar", "xf", fileName, *extract)
-		run("chmod", "a+x", *extract)
-		run("mv", "-f", *extract, *bindir+"/")
+		untar(fileName, *extract, *bindir+"/")
 	}
 }

bin/make_manual.py

@@ -36,6 +36,7 @@ docs = [
     "http.md",
     "hubic.md",
     "jottacloud.md",
+    "koofr.md",
     "mega.md",
     "azureblob.md",
     "onedrive.md",

bin/upload-github

@@ -29,7 +29,7 @@ github-release release \
     --name "rclone" \
     --description "Rclone - rsync for cloud storage. Sync files to and from many cloud storage providers."
 
-for build in `ls build | grep -v current`; do
+for build in `ls build | grep -v current | grep -v testbuilds`; do
     echo "Uploading ${build}"
     base="${build%.*}"
     parts=(${base//-/ })

cmd/cmd.go

@@ -230,22 +230,31 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	SigInfoHandler()
 	for try := 1; try <= *retries; try++ {
 		err = f()
-		if !Retry || (err == nil && !accounting.Stats.Errored()) {
+		fs.CountError(err)
+		if !Retry || !accounting.Stats.Errored() {
 			if try > 1 {
 				fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries)
 			}
 			break
 		}
-		if fserrors.IsFatalError(err) || accounting.Stats.HadFatalError() {
+		if accounting.Stats.HadFatalError() {
 			fs.Errorf(nil, "Fatal error received - not attempting retries")
 			break
 		}
-		if fserrors.IsNoRetryError(err) || (accounting.Stats.Errored() && !accounting.Stats.HadRetryError()) {
+		if accounting.Stats.Errored() && !accounting.Stats.HadRetryError() {
 			fs.Errorf(nil, "Can't retry this error - not attempting retries")
 			break
 		}
-		if err != nil {
-			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), err)
+		if retryAfter := accounting.Stats.RetryAfter(); !retryAfter.IsZero() {
+			d := retryAfter.Sub(time.Now())
+			if d > 0 {
+				fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
+				time.Sleep(d)
+			}
+		}
+		lastErr := accounting.Stats.GetLastError()
+		if lastErr != nil {
+			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), lastErr)
 		} else {
 			fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.Stats.GetErrors())
 		}
@@ -341,8 +350,7 @@ func initConfig() {
 	configflags.SetFlags()
 
 	// Load filters
-	var err error
-	filter.Active, err = filter.NewFilter(&filterflags.Opt)
+	err := filterflags.Reload()
 	if err != nil {
 		log.Fatalf("Failed to load filters: %v", err)
 	}
@@ -456,7 +464,7 @@ func AddBackendFlags() {
 					help = help[:nl]
 				}
 				help = strings.TrimSpace(help)
-				flag := pflag.CommandLine.VarPF(opt, name, string(opt.ShortOpt), help)
+				flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
 				if _, isBool := opt.Default.(bool); isBool {
 					flag.NoOptDefVal = "true"
 				}

cmd/cmount/fs.go

@@ -388,7 +388,11 @@ func (fsys *FS) Release(path string, fh uint64) (errc int) {
 		return errc
 	}
 	_ = fsys.closeHandle(fh)
-	return translateError(handle.Release())
+	// Run the Release asynchronously, ignoring errors
+	go func() {
+		_ = handle.Release()
+	}()
+	return 0
 }
 
 // Unlink removes a file.

cmd/cmount/mount.go

@@ -127,7 +127,7 @@ func waitFor(fn func() bool) (ok bool) {
 func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
-	// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
+	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
 	if runtime.GOOS != "windows" {
 		fi, err := os.Stat(mountpoint)
 		if err != nil {

cmd/copy/copy.go

@@ -7,8 +7,13 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	createEmptySrcDirs = false
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy")
 }
 
 var commandDefintion = &cobra.Command{
@@ -69,7 +74,7 @@ changed recently very efficiently like this:
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc)
+				return sync.CopyDir(fdst, fsrc, createEmptySrcDirs)
 			}
 			return operations.CopyFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/copyto/copyto.go

@@ -48,7 +48,7 @@ destination.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc)
+				return sync.CopyDir(fdst, fsrc, false)
 			}
 			return operations.CopyFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/help.go

@@ -32,8 +32,48 @@ documentation, changelog and configuration walkthroughs.
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
 	},
+	BashCompletionFunction: bashCompletionFunc,
 }
 
+const (
+	bashCompletionFunc = `
+__rclone_custom_func() {
+    if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
+        local cur cword prev words
+        if declare -F _init_completion > /dev/null; then
+            _init_completion -n : || return
+        else
+            __rclone_init_completion -n : || return
+        fi
+        if [[ $cur != *:* ]]; then
+            local remote
+            while IFS= read -r remote; do
+                [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
+            done < <(command rclone listremotes)
+            if [[ ${COMPREPLY[@]} ]]; then
+                local paths=("$cur"*)
+                [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
+            fi
+        else
+            local path=${cur#*:}
+            if [[ $path == */* ]]; then
+                local prefix=$(eval printf '%s' "${path%/*}")
+            else
+                local prefix=
+            fi
+            local line
+            while IFS= read -r line; do
+                local reply=${prefix:+$prefix/}$line
+                [[ $reply != $path* ]] || COMPREPLY+=("$reply")
+            done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
+	    [[ ! ${COMPREPLY[@]} ]] || compopt -o filenames
+        fi
+        [[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
+    fi
+}
+`
+)
+
 // root help command
 var helpCommand = &cobra.Command{
 	Use:   "help",

cmd/info/info.go

@@ -21,11 +21,22 @@ import (
 	"github.com/spf13/cobra"
 )
 
+type position int
+
+const (
+	positionMiddle position = 1 << iota
+	positionLeft
+	positionRight
+	positionNone position = 0
+	positionAll  position = positionRight<<1 - 1
+)
+
 var (
 	checkNormalization bool
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
+	positionList       = []position{positionMiddle, positionLeft, positionRight}
 )
 
 func init() {
@@ -59,7 +70,7 @@ a bit of go code for each one.
 type results struct {
 	f                    fs.Fs
 	mu                   sync.Mutex
-	charNeedsEscaping    map[rune]bool
+	stringNeedsEscaping  map[string]position
 	maxFileLength        int
 	canWriteUnnormalized bool
 	canReadUnnormalized  bool
@@ -69,8 +80,8 @@ type results struct {
 
 func newResults(f fs.Fs) *results {
 	return &results{
-		f:                 f,
-		charNeedsEscaping: make(map[rune]bool),
+		f:                   f,
+		stringNeedsEscaping: make(map[string]position),
 	}
 }
 
@@ -79,13 +90,13 @@ func (r *results) Print() {
 	fmt.Printf("// %s\n", r.f.Name())
 	if checkControl {
 		escape := []string{}
-		for c, needsEscape := range r.charNeedsEscaping {
-			if needsEscape {
+		for c, needsEscape := range r.stringNeedsEscaping {
+			if needsEscape != positionNone {
 				escape = append(escape, fmt.Sprintf("0x%02X", c))
 			}
 		}
 		sort.Strings(escape)
-		fmt.Printf("charNeedsEscaping = []byte{\n")
+		fmt.Printf("stringNeedsEscaping = []byte{\n")
 		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
 		fmt.Printf("}\n")
 	}
@@ -130,20 +141,45 @@ func (r *results) checkUTF8Normalization() {
 	}
 }
 
-// check we can write file with the rune passed in
-func (r *results) checkChar(c rune) {
-	fs.Infof(r.f, "Writing file 0x%02X", c)
-	path := fmt.Sprintf("0x%02X-%c-", c, c)
-	_, err := r.writeFile(path)
-	escape := false
-	if err != nil {
-		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
-		escape = true
-	} else {
-		fs.Infof(r.f, "OK writing file 0x%02X", c)
+func (r *results) checkStringPositions(s string) {
+	fs.Infof(r.f, "Writing position file 0x%0X", s)
+	positionError := positionNone
+
+	for _, pos := range positionList {
+		path := ""
+		switch pos {
+		case positionMiddle:
+			path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
+		case positionLeft:
+			path = fmt.Sprintf("%s-position-left-%0X", s, s)
+		case positionRight:
+			path = fmt.Sprintf("position-right-%0X-%s", s, s)
+		default:
+			panic("invalid position: " + pos.String())
+		}
+		_, writeErr := r.writeFile(path)
+		if writeErr != nil {
+			fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
+		} else {
+			fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
+		}
+		obj, getErr := r.f.NewObject(path)
+		if getErr != nil {
+			fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
+		} else {
+			if obj.Size() != 50 {
+				fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
+			} else {
+				fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
+			}
+		}
+		if writeErr != nil || getErr != nil {
+			positionError += pos
+		}
 	}
+
 	r.mu.Lock()
-	r.charNeedsEscaping[c] = escape
+	r.stringNeedsEscaping[s] = positionError
 	r.mu.Unlock()
 }
 
@@ -157,19 +193,28 @@ func (r *results) checkControls() {
 	}
 	var wg sync.WaitGroup
 	for i := rune(0); i < 128; i++ {
+		s := string(i)
 		if i == 0 || i == '/' {
 			// We're not even going to check NULL or /
-			r.charNeedsEscaping[i] = true
+			r.stringNeedsEscaping[s] = positionAll
 			continue
 		}
 		wg.Add(1)
-		c := i
-		go func() {
+		go func(s string) {
 			defer wg.Done()
 			token := <-tokens
-			r.checkChar(c)
+			r.checkStringPositions(s)
 			tokens <- token
-		}()
+		}(s)
+	}
+	for _, s := range []string{"＼", "\xBF", "\xFE"} {
+		wg.Add(1)
+		go func(s string) {
+			defer wg.Done()
+			token := <-tokens
+			r.checkStringPositions(s)
+			tokens <- token
+		}(s)
 	}
 	wg.Wait()
 	fs.Infof(r.f, "Done trying to create control character file names")
@@ -268,3 +313,35 @@ func readInfo(f fs.Fs) error {
 	r.Print()
 	return nil
 }
+
+func (e position) String() string {
+	switch e {
+	case positionNone:
+		return "none"
+	case positionAll:
+		return "all"
+	}
+	var buf bytes.Buffer
+	if e&positionMiddle != 0 {
+		buf.WriteString("middle")
+		e &= ^positionMiddle
+	}
+	if e&positionLeft != 0 {
+		if buf.Len() != 0 {
+			buf.WriteRune(',')
+		}
+		buf.WriteString("left")
+		e &= ^positionLeft
+	}
+	if e&positionRight != 0 {
+		if buf.Len() != 0 {
+			buf.WriteRune(',')
+		}
+		buf.WriteString("right")
+		e &= ^positionRight
+	}
+	if e != positionNone {
+		panic("invalid position")
+	}
+	return buf.String()
+}

cmd/info/process.sh

@@ -0,0 +1,40 @@
+set -euo pipefail
+
+for f in info-*.log; do 
+  for pos in middle left right; do
+    egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos
+    egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos
+  done
+  {
+    echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}"
+    echo "Write\tWrite\tWrite\tGet\tGet\tGet"
+    echo "Mid\tLeft\tRight\tMid\tLeft\tRight"
+    paste $f.write_{middle,left,right} $f.get_{middle,left,right}
+  } > $f.csv
+done
+
+for f in info-*.list; do
+  for pos in middle left right; do
+    cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos
+  done
+  {
+    echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}"
+    echo "List\tList\tList"
+    echo "Mid\tLeft\tRight"
+    for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do
+      echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss)
+      echo -n "\t"
+      echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/'    $f.left   | tr -d "\t\r" | grep -a . || echo Miss)
+      echo -n "\t"
+      echo    $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/'   $f.right  | tr -d "\t\r" | grep -a . || echo Miss)
+      # echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t"
+      # echo -n $(grep -a "position-left-$e"    $f.left   | tr -d "\t\r" || echo Miss)"\t"
+      # echo    $(grep -a "position-right-$e-"  $f.right  | tr -d "\t\r" || echo Miss)
+    done
+    } > $f.csv
+done
+
+for f in info-*.list; do 
+    paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv
+done
+paste *.full.csv > info-complete.csv

cmd/info/test.cmd

@@ -0,0 +1,3 @@
+rclone.exe purge    info
+rclone.exe info -vv info > info-LocalWindows.log  2>&1
+rclone.exe ls   -vv info > info-LocalWindows.list 2>&1

cmd/info/test.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env zsh
+#
+# example usage: 
+# $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh --list | \
+#   parallel -P20 $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh
+
+export PATH=$GOPATH/src/github.com/ncw/rclone:$PATH
+
+typeset -A allRemotes
+ allRemotes=(
+  TestAmazonCloudDrive '--low-level-retries=2 --checkers=5'
+  TestB2 ''
+  TestBox ''
+  TestDrive '--tpslimit=5'
+  TestCrypt ''
+  TestDropbox '--checkers=1'
+  TestJottacloud ''
+  TestMega ''
+  TestOneDrive ''
+  TestOpenDrive '--low-level-retries=2 --checkers=5'
+  TestPcloud '--low-level-retries=2 --timeout=15s'
+  TestS3 ''
+  Local ''
+)
+
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+ set -- ${(k)allRemotes[@]}
+elif [[ $1 = --list ]]; then
+  printf '%s\n' ${(k)allRemotes[@]}
+  exit 0
+fi
+
+for remote; do
+  dir=$remote:infotest
+  if [[ $remote = Local ]]; then
+    dir=infotest
+  fi
+  rclone purge    $dir || :
+  rclone info -vv $dir ${=allRemotes[$remote]} &> info-$remote.log
+  rclone ls   -vv $dir &> info-$remote.list
+done

cmd/listremotes/listremotes.go

@@ -16,7 +16,7 @@ var (
 
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
-	commandDefintion.Flags().BoolVarP(&listLong, "long", "l", listLong, "Show the type as well as names.")
+	commandDefintion.Flags().BoolVarP(&listLong, "long", "", listLong, "Show the type as well as names.")
 }
 
 var commandDefintion = &cobra.Command{

cmd/ls/lshelp/lshelp.go

@@ -2,7 +2,7 @@ package lshelp
 
 // Help describes the common help for all the list commands
 var Help = `
-Any of the filtering options can be applied to this commmand.
+Any of the filtering options can be applied to this command.
 
 There are several related list commands
 

cmd/lsf/lsf.go

@@ -10,7 +10,6 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/operations"
-	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -67,8 +66,11 @@ output:
     s - size
     t - modification time
     h - hash
-    i - ID of object if known
+    i - ID of object
+    o - Original ID of underlying object
     m - MimeType of object if known
+    e - encrypted name
+    T - tier of storage if known, eg "Hot" or "Cool"
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -161,6 +163,12 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 	list.SetCSV(csv)
 	list.SetDirSlash(dirSlash)
 	list.SetAbsolute(absolute)
+	var opt = operations.ListJSONOpt{
+		NoModTime: true,
+		DirsOnly:  dirsOnly,
+		FilesOnly: filesOnly,
+		Recurse:   recurse,
+	}
 
 	for _, char := range format {
 		switch char {
@@ -168,38 +176,31 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 			list.AddPath()
 		case 't':
 			list.AddModTime()
+			opt.NoModTime = false
 		case 's':
 			list.AddSize()
 		case 'h':
 			list.AddHash(hashType)
+			opt.ShowHash = true
 		case 'i':
 			list.AddID()
 		case 'm':
 			list.AddMimeType()
+		case 'e':
+			list.AddEncrypted()
+			opt.ShowEncrypted = true
+		case 'o':
+			list.AddOrigID()
+			opt.ShowOrigIDs = true
+		case 'T':
+			list.AddTier()
 		default:
 			return errors.Errorf("Unknown format character %q", char)
 		}
 	}
 
-	return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error {
-		if err != nil {
-			fs.CountError(err)
-			fs.Errorf(path, "error listing: %v", err)
-			return nil
-		}
-		for _, entry := range entries {
-			_, isDir := entry.(fs.Directory)
-			if isDir {
-				if filesOnly {
-					continue
-				}
-			} else {
-				if dirsOnly {
-					continue
-				}
-			}
-			_, _ = fmt.Fprintln(out, list.Format(entry))
-		}
+	return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
+		_, _ = fmt.Fprintln(out, list.Format(item))
 		return nil
 	})
 }

cmd/lsjson/lsjson.go

@@ -23,6 +23,8 @@ func init() {
 	commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
 	commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.")
 	commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
+	commandDefintion.Flags().BoolVarP(&opt.FilesOnly, "files-only", "", false, "Show only files in the listing.")
+	commandDefintion.Flags().BoolVarP(&opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing.")
 }
 
 var commandDefintion = &cobra.Command{
@@ -55,12 +57,22 @@ If --no-modtime is specified then ModTime will be blank.
 
 If --encrypted is not specified the Encrypted won't be emitted.
 
+If --dirs-only is not specified files in addition to directories are returned
+
+If --files-only is not specified directories in addition to the files will be returned.
+
 The Path field will only show folders below the remote path being listed.
 If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
 will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
 When used without --recursive the Path will always be the same as Name.
 
-The time is in RFC3339 format with nanosecond precision.
+The time is in RFC3339 format with up to nanosecond precision.  The
+number of decimal digits in the seconds will depend on the precision
+that the remote can hold the times, so if times are accurate to the
+nearest millisecond (eg Google Drive) then 3 digits will always be
+shown ("2017-05-31T16:15:57.034+01:00") whereas if the times are
+accurate to the nearest second (Dropbox, Box, WebDav etc) no digits
+will be shown ("2017-05-31T16:15:57+01:00").
 
 The whole output can be processed as a JSON blob, or alternatively it
 can be processed line by line as each item is written one to a line.

cmd/mount/dir.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // Dir represents a directory entry
@@ -20,7 +20,7 @@ type Dir struct {
 	*vfs.Dir
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.Node = (*Dir)(nil)
 
 // Attr updates the attributes of a directory

cmd/mount/file.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"io"
 	"time"
 
@@ -11,7 +12,6 @@ import (
 	"github.com/ncw/rclone/cmd/mountlib"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // File represents a file

cmd/mount/fs.go

@@ -5,6 +5,7 @@
 package mount
 
 import (
+	"context"
 	"syscall"
 
 	"bazil.org/fuse"
@@ -15,7 +16,6 @@ import (
 	"github.com/ncw/rclone/vfs"
 	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FS represents the top level filing system
@@ -24,7 +24,7 @@ type FS struct {
 	f fs.Fs
 }
 
-// Check interface satistfied
+// Check interface satisfied
 var _ fusefs.FS = (*FS)(nil)
 
 // NewFS makes a new FS
@@ -46,7 +46,7 @@ func (f *FS) Root() (node fusefs.Node, err error) {
 	return &Dir{root}, nil
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.FSStatfser = (*FS)(nil)
 
 // Statfs is called to obtain file system metadata.

cmd/mount/handle.go

@@ -3,13 +3,13 @@
 package mount
 
 import (
+	"context"
 	"io"
 
 	"bazil.org/fuse"
 	fusefs "bazil.org/fuse/fs"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FileHandle is an open for read file handle on a File
@@ -45,7 +45,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	if err != nil {
 		return translateError(err)
 	}
-	resp.Size = int(n)
+	resp.Size = n
 	return nil
 }
 
@@ -80,5 +80,9 @@ var _ fusefs.HandleReleaser = (*FileHandle)(nil)
 // the kernel
 func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) (err error) {
 	defer log.Trace(fh, "")("err=%v", &err)
-	return translateError(fh.Handle.Release())
+	// Run the Release asynchronously, ignoring errors
+	go func() {
+		_ = fh.Handle.Release()
+	}()
+	return nil
 }