build: make arm .deb packages be armel architecture

This is as recommended by debian.

Fixes #5107

.github/workflows/build.yml

@@ -19,12 +19,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'go1.13', 'go1.14', 'go1.15']
+        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.13', 'go1.14', 'go1.15']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -32,18 +32,25 @@ jobs:
             racequicktest: true
             deploy: true
 
-          - job_name: mac
+          - job_name: mac_amd64
             os: macOS-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
             racequicktest: true
             deploy: true
 
+          - job_name: mac_arm64
+            os: macOS-latest
+            go: '1.16.x'
+            gotags: 'cmount'
+            build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -macos-sdk macosx11.1 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
+            deploy: true
+
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             build_args: '-buildmode exe'
@@ -53,7 +60,7 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -64,8 +71,8 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.16.0-rc1'
-            build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
+            go: '1.16.x'
+            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
@@ -124,7 +131,7 @@ jobs:
         shell: bash
         run: |
           brew update
-          brew install --cask osxfuse
+          brew install --cask macfuse
         if: matrix.os == 'macOS-latest'
 
       - name: Install Libraries on Windows
@@ -237,7 +244,7 @@ jobs:
           #     .
           xgo \
               -image=billziss/xgo-cgofuse \
-              -targets=android/*,ios/* \
+              -targets=android/* \
               -dest build \
               .
 

.gitignore

@@ -1,6 +1,7 @@
 *~
 _junk/
 rclone
+rclone.exe
 build
 docs/public
 rclone.iml

CONTRIBUTING.md

@@ -72,7 +72,7 @@ Make sure you
 
 When you are done with that
 
-    git push origin my-new-feature
+    git push -u origin my-new-feature
 
 Go to the GitHub website and click [Create pull
 request](https://help.github.com/articles/creating-a-pull-request/).
@@ -99,7 +99,7 @@ rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
 
     go test -v ./...
-    
+
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
 storage systems by mocking all their interfaces, rclone unit tests can
@@ -115,8 +115,8 @@ are skipped if `TestDrive:` isn't defined.
     cd backend/drive
     go test -v
 
-You can then run the integration tests which tests all of rclone's
-operations.  Normally these get run against the local filing system,
+You can then run the integration tests which test all of rclone's
+operations.  Normally these get run against the local file system,
 but they can be run against any of the remotes.
 
     cd fs/sync
@@ -127,7 +127,7 @@ but they can be run against any of the remotes.
     go test -v -remote TestDrive:
 
 If you want to use the integration test framework to run these tests
-all together with an HTML report and test retries then from the
+altogether with an HTML report and test retries then from the
 project root:
 
     go install github.com/rclone/rclone/fstest/test_all
@@ -202,7 +202,7 @@ for the flag help, the remainder is shown to the user in `rclone
 config` and is added to the docs with `make backenddocs`.
 
 The only documentation you need to edit are the `docs/content/*.md`
-files.  The MANUAL.*, rclone.1, web site, etc. are all auto generated
+files.  The `MANUAL.*`, `rclone.1`, web site, etc. are all auto generated
 from those during the release process.  See the `make doc` and `make
 website` targets in the Makefile if you are interested in how.  You
 don't need to run these when adding a feature.
@@ -265,7 +265,7 @@ rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-rclone can be built with modules outside of the GOPATH
+rclone can be built with modules outside of the `GOPATH`.
 
 To add a dependency `github.com/ncw/new_dependency` see the
 instructions below.  These will fetch the dependency and add it to
@@ -333,8 +333,8 @@ Getting going
   * Try to implement as many optional methods as possible as it makes the remote more usable.
   * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
     * `rclone purge -v TestRemote:rclone-info`
-    * `rclone info --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    * `go run cmd/info/internal/build_csv/main.go -o remote.csv remote.json`
+    * `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    * `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
     * open `remote.csv` in a spreadsheet and examine
 
 Unit tests
@@ -400,7 +400,7 @@ Usage
    - If this variable doesn't exist, plugin support is disabled.
    - Plugins must be compiled against the exact version of rclone to work.
      (The rclone used during building the plugin must be the same as the source of rclone)
-     
+
 Building
 
 To turn your existing additions into a Go plugin, move them to an external repository

Dockerfile

@@ -16,6 +16,8 @@ RUN apk --no-cache add ca-certificates fuse tzdata && \
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 
+RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
+
 ENTRYPOINT [ "rclone" ]
 
 WORKDIR /data

MANUAL.html

@@ -1471,11 +1471,11 @@ rclone mount remote:path/to/files * --volname \\cloud\remote</code></pre>
 <p>Note that drives created as Administrator are not visible by other accounts (including the account that was elevated as Administrator). So if you start a Windows drive from an Administrative Command Prompt and then try to access the same drive from Explorer (which does not run as Administrator), you will not be able to see the new drive.</p>
 <p>The easiest way around this is to start the drive from a normal command prompt. It is also possible to start a drive from the SYSTEM account (using <a href="https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture">the WinFsp.Launcher infrastructure</a>) which creates drives accessible for everyone on the system or alternatively using <a href="https://nssm.cc/usage">the nssm service manager</a>.</p>
 <h2 id="limitations">Limitations</h2>
-<p>Without the use of <code>--vfs-cache-mode</code> this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without <code>--vfs-cache-mode writes</code> or <code>--vfs-cache-mode full</code>. See the <a href="#file-caching">File Caching</a> section for more info.</p>
+<p>Without the use of <code>--vfs-cache-mode</code> this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without <code>--vfs-cache-mode writes</code> or <code>--vfs-cache-mode full</code>. See the <a href="#vfs-file-caching">VFS File Caching</a> section for more info.</p>
 <p>The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2, Hubic) do not support the concept of empty directories, so empty directories will have a tendency to disappear once they fall out of the directory cache.</p>
 <p>Only supported on Linux, FreeBSD, OS X and Windows at the moment.</p>
 <h2 id="rclone-mount-vs-rclone-synccopy">rclone mount vs rclone sync/copy</h2>
-<p>File systems expect things to be 100% reliable, whereas cloud storage systems are a long way from 100% reliable. The rclone sync/copy commands cope with this with lots of retries. However rclone mount can't use retries in the same way without making local copies of the uploads. Look at the <a href="#file-caching">file caching</a> for solutions to make mount more reliable.</p>
+<p>File systems expect things to be 100% reliable, whereas cloud storage systems are a long way from 100% reliable. The rclone sync/copy commands cope with this with lots of retries. However rclone mount can't use retries in the same way without making local copies of the uploads. Look at the <a href="#vfs-file-caching">VFS file caching</a> section for solutions to make mount more reliable.</p>
 <h2 id="attribute-caching">Attribute caching</h2>
 <p>You can use the flag <code>--attr-timeout</code> to set the time the kernel caches the attributes (size, modification time, etc.) for directory entries.</p>
 <p>The default is <code>1s</code> which caches files just long enough to avoid too many callbacks to rclone from the kernel.</p>

MANUAL.md

@@ -2928,7 +2928,7 @@ Without the use of `--vfs-cache-mode` this can only write files
 sequentially, it can only seek when reading.  This means that many
 applications won't work with their files on an rclone mount without
 `--vfs-cache-mode writes` or `--vfs-cache-mode full`.
-See the [File Caching](#file-caching) section for more info.
+See the [VFS File Caching](#vfs-file-caching) section for more info.
 
 The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2,
 Hubic) do not support the concept of empty directories, so empty
@@ -2943,7 +2943,7 @@ File systems expect things to be 100% reliable, whereas cloud storage
 systems are a long way from 100% reliable. The rclone sync/copy
 commands cope with this with lots of retries.  However rclone mount
 can't use retries in the same way without making local copies of the
-uploads. Look at the [file caching](#file-caching)
+uploads. Look at the [VFS File Caching](#vfs-file-caching)
 for solutions to make mount more reliable.
 
 ## Attribute caching

Makefile

@@ -93,7 +93,7 @@ build_dep:
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
-	cd /tmp && go get github.com/goreleaser/nfpm/v2/...
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64\.tar\.gz'
 
 # Get the release dependencies we only install on Windows
 release_dep_windows:
@@ -119,7 +119,7 @@ doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 rclone.1:	MANUAL.md
 	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md

backend/alias/alias_internal_test.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local" // pull in test backend
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/stretchr/testify/require"
 )
 
@@ -19,7 +20,7 @@ var (
 )
 
 func prepare(t *testing.T, root string) {
-	config.LoadConfig(context.Background())
+	configfile.LoadConfig(context.Background())
 
 	// Configure the remote
 	config.FileSet(remoteName, "type", "alias")

backend/amazonclouddrive/amazonclouddrive.go

@@ -205,7 +205,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if resp != nil {
 		if resp.StatusCode == 401 {
 			f.tokenRenewer.Invalidate()
@@ -280,7 +283,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.getRootInfo()
+		_, err := f.getRootInfo(ctx)
 		return err
 	})
 
@@ -288,14 +291,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		_, resp, err = f.c.Account.GetEndpoints()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get endpoints")
 	}
 
 	// Get rootID
-	rootInfo, err := f.getRootInfo()
+	rootInfo, err := f.getRootInfo(ctx)
 	if err != nil || rootInfo.Id == nil {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
@@ -337,11 +340,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // getRootInfo gets the root folder info
-func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) {
+func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		rootInfo, resp, err = f.c.Nodes.GetRoot()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return rootInfo, err
 }
@@ -380,7 +383,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	var subFolder *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -407,7 +410,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var info *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -428,7 +431,7 @@ type listAllFn func(*acd.Node) bool
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	query := "parents:" + dirID
 	if directoriesOnly {
 		query += " AND kind:" + folderKind
@@ -449,7 +452,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 		var resp *http.Response
 		err = f.pacer.CallNoRetry(func() (bool, error) {
 			nodes, resp, err = f.c.Nodes.GetNodes(&opts)
-			return f.shouldRetry(resp, err)
+			return f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return false, err
@@ -508,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var iErr error
 	for tries := 1; tries <= maxTries; tries++ {
 		entries = nil
-		_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool {
 			remote := path.Join(dir, *node.Name)
 			switch *node.Kind {
 			case folderKind:
@@ -667,7 +670,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		if ok {
 			return false, nil
 		}
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -708,7 +711,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
+	err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
 	if err != nil {
 		return nil, err
 	}
@@ -803,7 +806,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var jsonStr string
 	err = srcFs.pacer.Call(func() (bool, error) {
 		jsonStr, err = srcInfo.GetMetadata()
-		return srcFs.shouldRetry(nil, err)
+		return srcFs.shouldRetry(ctx, nil, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error: error reading src metadata: %v", err)
@@ -815,7 +818,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return err
 	}
 
-	err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
+	err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
 	if err != nil {
 		return err
 	}
@@ -840,7 +843,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if check {
 		// check directory is empty
 		empty := true
-		_, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool {
 			switch *node.Kind {
 			case folderKind:
 				empty = false
@@ -865,7 +868,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = node.Trash()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -987,7 +990,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	var info *acd.File
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -1044,7 +1047,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		} else {
 			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	return in, err
 }
@@ -1067,7 +1070,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if ok {
 			return false, nil
 		}
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1077,70 +1080,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove a node
-func (f *Fs) removeNode(info *acd.Node) error {
+func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error {
 	var resp *http.Response
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = info.Trash()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return err
 }
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.removeNode(o.info)
+	return o.fs.removeNode(ctx, o.info)
 }
 
 // Restore a node
-func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
+func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Restore()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return newInfo, err
 }
 
 // Changes name of given node
-func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
+func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return newInfo, err
 }
 
 // Replaces one parent with another, effectively moving the file. Leaves other
 // parents untouched. ReplaceParent cannot be used when the file is trashed.
-func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error {
+func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.ReplaceParent(oldParentID, newParentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // Adds one additional parent to object.
-func (f *Fs) addParent(info *acd.Node, newParentID string) error {
+func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.AddParent(newParentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // Remove given parent from object, leaving the other possible
 // parents untouched. Object can end up having no parents.
-func (f *Fs) removeParent(info *acd.Node, parentID string) error {
+func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.RemoveParent(parentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // moveNode moves the node given from the srcLeaf,srcDirectoryID to
 // the dstLeaf,dstDirectoryID
-func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
+func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
 	// fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID)
 	cantMove := fs.ErrorCantMove
 	if useDirErrorMsgs {
@@ -1154,7 +1157,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
 
 	if srcLeaf != dstLeaf {
 		// fs.Debugf(name, "renaming")
-		_, err = f.renameNode(srcInfo, dstLeaf)
+		_, err = f.renameNode(ctx, srcInfo, dstLeaf)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path rename failed: %v", err)
 			goto OnConflict
@@ -1162,7 +1165,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
 	}
 	if srcDirectoryID != dstDirectoryID {
 		// fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID)
-		err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID)
+		err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path parent replace failed: %v", err)
 			return err
@@ -1175,13 +1178,13 @@ OnConflict:
 	fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.")
 
 	// fs.Debugf(name, "Trashing file")
-	err = f.removeNode(srcInfo)
+	err = f.removeNode(ctx, srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: remove node failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Renaming file")
-	_, err = f.renameNode(srcInfo, dstLeaf)
+	_, err = f.renameNode(ctx, srcInfo, dstLeaf)
 	if err != nil {
 		fs.Debugf(name, "Move: rename node failed: %v", err)
 		return err
@@ -1189,19 +1192,19 @@ OnConflict:
 	// note: replacing parent is forbidden by API, modifying them individually is
 	// okay though
 	// fs.Debugf(name, "Adding target parent")
-	err = f.addParent(srcInfo, dstDirectoryID)
+	err = f.addParent(ctx, srcInfo, dstDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: addParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "removing original parent")
-	err = f.removeParent(srcInfo, srcDirectoryID)
+	err = f.removeParent(ctx, srcInfo, srcDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: removeParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Restoring")
-	_, err = f.restoreNode(srcInfo)
+	_, err = f.restoreNode(ctx, srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: restoreNode node failed: %v", err)
 		return err

backend/azureblob/azureblob.go

@@ -347,7 +347,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// FIXME interpret special errors - more to do here
 	if storageErr, ok := err.(azblob.StorageError); ok {
 		switch storageErr.ServiceCode() {
@@ -578,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// Retry as specified by the documentation:
 			// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
 			token, err = GetMSIToken(ctx, userMSI)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 
 		if err != nil {
@@ -594,7 +597,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			var refreshedToken adal.Token
 			err := f.imdsPacer.Call(func() (bool, error) {
 				refreshedToken, err = GetMSIToken(ctx, userMSI)
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			})
 			if err != nil {
 				// Failed to refresh.
@@ -803,7 +806,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 
 		if err != nil {
@@ -1029,7 +1032,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return err
@@ -1098,7 +1101,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 					}
 				}
 			}
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 	}, nil)
 }
@@ -1136,10 +1139,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error {
 					return false, fs.ErrorDirNotFound
 				}
 
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			}
 
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 	})
 }
@@ -1212,7 +1215,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	err = f.pacer.Call(func() (bool, error) {
 		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1373,7 +1376,7 @@ func (o *Object) readMetaData() (err error) {
 	var blobProperties *azblob.BlobGetPropertiesResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
@@ -1408,7 +1411,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	blob := o.getBlobReference()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1451,7 +1454,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var downloadResponse *azblob.DownloadResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open for download")
@@ -1592,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Stream contents of the reader object to the given blob URL
 		blockBlobURL := blob.ToBlockBlobURL()
 		_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1620,7 +1623,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	ac := azblob.BlobAccessConditions{}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.Delete(ctx, snapShotOptions, ac)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 }
 
@@ -1649,7 +1652,7 @@ func (o *Object) SetTier(tier string) error {
 	ctx := context.Background()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 
 	if err != nil {

backend/b2/b2.go

@@ -305,7 +305,10 @@ var retryErrorCodes = []int{
 
 // shouldRetryNoAuth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// For 429 or 503 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.
@@ -336,7 +339,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 		}
 		return true, err
 	}
-	return f.shouldRetryNoReauth(resp, err)
+	return f.shouldRetryNoReauth(ctx, resp, err)
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -479,12 +482,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.setRoot(newRoot)
 		_, err := f.NewObject(ctx, leaf)
 		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				// File doesn't exist so return old f
-				f.setRoot(oldRoot)
-				return f, nil
-			}
-			return nil, err
+			// File doesn't exist so return old f
+			f.setRoot(oldRoot)
+			return f, nil
 		}
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
@@ -507,7 +507,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
-		return f.shouldRetryNoReauth(resp, err)
+		return f.shouldRetryNoReauth(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to authenticate")

backend/box/box.go

@@ -317,7 +317,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
@@ -548,7 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -585,7 +588,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -740,7 +743,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -767,7 +770,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -839,7 +842,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -877,7 +880,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -895,7 +898,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read user info")
@@ -1008,7 +1011,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return info.SharedLink.URL, err
 }
@@ -1026,7 +1029,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1048,7 +1051,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list trash")
@@ -1182,7 +1185,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return info, err
 }
@@ -1215,7 +1218,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1255,7 +1258,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err

backend/box/upload.go

@@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return
 }
@@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -109,10 +109,10 @@ outer:
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
-				return shouldRetry(resp, err)
+				return shouldRetry(ctx, resp, err)
 			}
 			body, err = rest.ReadBody(resp)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		delay := defaultDelay
 		var why string
@@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }

backend/cache/cache_internal_test.go

@@ -892,7 +892,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.FileGet(remote, "type", "")
+		remoteType := config.FileGet(remote, "type")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -903,14 +903,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.FileGet(remote, "remote", "")
+			remoteRemote := config.FileGet(remote, "remote")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.FileGet(remoteWrapping, "type", "")
+			remoteType := config.FileGet(remoteWrapping, "type")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -1034,7 +1034,7 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
 	objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
 	objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
 
-	obj, err = f.Put(context.Background(), in1, objInfo1)
+	_, err = f.Put(context.Background(), in1, objInfo1)
 	require.NoError(t, err)
 	obj, err = f.NewObject(context.Background(), remote)
 	require.NoError(t, err)

backend/chunker/chunker.go

@@ -47,7 +47,8 @@ import (
 // The following types of chunks are supported:
 // data and control, active and temporary.
 // Chunk type is identified by matching chunk file name
-// based on the chunk name format configured by user.
+// based on the chunk name format configured by user and transaction
+// style being used.
 //
 // Both data and control chunks can be either temporary (aka hidden)
 // or active (non-temporary aka normal aka permanent).
@@ -63,6 +64,12 @@ import (
 // which is transparently converted to the new format. In its maximum
 // length of 13 decimals it makes a 7-digit base-36 number.
 //
+// When transactions is set to the norename style, data chunks will
+// keep their temporary chunk names (with the transacion identifier
+// suffix). To distinguish them from temporary chunks, the txn field
+// of the metadata file is set to match the transaction identifier of
+// the data chunks.
+//
 // Chunker can tell data chunks from control chunks by the characters
 // located in the "hash placeholder" position of configured format.
 // Data chunks have decimal digits there.
@@ -101,7 +108,7 @@ const maxMetadataSize = 1023
 const maxMetadataSizeWritten = 255
 
 // Current/highest supported metadata format.
-const metadataVersion = 1
+const metadataVersion = 2
 
 // optimizeFirstChunk enables the following optimization in the Put:
 // If a single chunk is expected, put the first chunk using the
@@ -224,6 +231,31 @@ It has the following fields: ver, size, nchunks, md5, sha1.`,
 					Help:  "Warn user, skip incomplete file and proceed.",
 				},
 			},
+		}, {
+			Name:     "transactions",
+			Advanced: true,
+			Default:  "rename",
+			Help:     `Choose how chunker should handle temporary files during transactions.`,
+			Hide:     fs.OptionHideCommandLine,
+			Examples: []fs.OptionExample{
+				{
+					Value: "rename",
+					Help:  "Rename temporary files after a successful transaction.",
+				}, {
+					Value: "norename",
+					Help: `Leave temporary file names and write transaction ID to metadata file.
+Metadata is required for no rename transactions (meta format cannot be "none").
+If you are using norename transactions you should be careful not to downgrade Rclone
+as older versions of Rclone don't support this transaction style and will misinterpret
+files manipulated by norename transactions.
+This method is EXPERIMENTAL, don't use on production systems.`,
+				}, {
+					Value: "auto",
+					Help: `Rename or norename will be used depending on capabilities of the backend.
+If meta format is set to "none", rename transactions will always be used.
+This method is EXPERIMENTAL, don't use on production systems.`,
+				},
+			},
 		}},
 	})
 }
@@ -271,7 +303,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
-	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType); err != nil {
+	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType, opt.Transactions); err != nil {
 		return nil, err
 	}
 
@@ -309,13 +341,14 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Remote     string        `config:"remote"`
-	ChunkSize  fs.SizeSuffix `config:"chunk_size"`
-	NameFormat string        `config:"name_format"`
-	StartFrom  int           `config:"start_from"`
-	MetaFormat string        `config:"meta_format"`
-	HashType   string        `config:"hash_type"`
-	FailHard   bool          `config:"fail_hard"`
+	Remote       string        `config:"remote"`
+	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
+	NameFormat   string        `config:"name_format"`
+	StartFrom    int           `config:"start_from"`
+	MetaFormat   string        `config:"meta_format"`
+	HashType     string        `config:"hash_type"`
+	FailHard     bool          `config:"fail_hard"`
+	Transactions string        `config:"transactions"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -337,12 +370,13 @@ type Fs struct {
 	opt          Options        // copy of Options
 	features     *fs.Features   // optional features
 	dirSort      bool           // reserved for future, ignored
+	useNoRename  bool           // can be set with the transactions option
 }
 
 // configure sets up chunker for given name format, meta format and hash type.
 // It also seeds the source of random transaction identifiers.
 // configure must be called only from NewFs or by unit tests.
-func (f *Fs) configure(nameFormat, metaFormat, hashType string) error {
+func (f *Fs) configure(nameFormat, metaFormat, hashType, transactionMode string) error {
 	if err := f.setChunkNameFormat(nameFormat); err != nil {
 		return errors.Wrapf(err, "invalid name format '%s'", nameFormat)
 	}
@@ -352,6 +386,9 @@ func (f *Fs) configure(nameFormat, metaFormat, hashType string) error {
 	if err := f.setHashType(hashType); err != nil {
 		return err
 	}
+	if err := f.setTransactionMode(transactionMode); err != nil {
+		return err
+	}
 
 	randomSeed := time.Now().UnixNano()
 	f.xactIDRand = rand.New(rand.NewSource(randomSeed))
@@ -411,6 +448,27 @@ func (f *Fs) setHashType(hashType string) error {
 	return nil
 }
 
+func (f *Fs) setTransactionMode(transactionMode string) error {
+	switch transactionMode {
+	case "rename":
+		f.useNoRename = false
+	case "norename":
+		if !f.useMeta {
+			return errors.New("incompatible transaction options")
+		}
+		f.useNoRename = true
+	case "auto":
+		f.useNoRename = !f.CanQuickRename()
+		if f.useNoRename && !f.useMeta {
+			f.useNoRename = false
+			return errors.New("using norename transactions requires metadata")
+		}
+	default:
+		return fmt.Errorf("unsupported transaction mode '%s'", transactionMode)
+	}
+	return nil
+}
+
 // setChunkNameFormat converts pattern based chunk name format
 // into Printf format and Regular expressions for data and
 // control chunks.
@@ -693,6 +751,7 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 	byRemote := make(map[string]*Object)
 	badEntry := make(map[string]bool)
 	isSubdir := make(map[string]bool)
+	txnByRemote := map[string]string{}
 
 	var tempEntries fs.DirEntries
 	for _, dirOrObject := range sortedEntries {
@@ -705,12 +764,18 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 				object := f.newObject("", entry, nil)
 				byRemote[remote] = object
 				tempEntries = append(tempEntries, object)
+				if f.useNoRename {
+					txnByRemote[remote], err = object.readXactID(ctx)
+					if err != nil {
+						return nil, err
+					}
+				}
 				break
 			}
 			// this is some kind of chunk
 			// metobject should have been created above if present
-			isSpecial := xactID != "" || ctrlType != ""
 			mainObject := byRemote[mainRemote]
+			isSpecial := xactID != txnByRemote[mainRemote] || ctrlType != ""
 			if mainObject == nil && f.useMeta && !isSpecial {
 				fs.Debugf(f, "skip orphan data chunk %q", remote)
 				break
@@ -809,10 +874,11 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 	}
 
 	var (
-		o        *Object
-		baseObj  fs.Object
-		err      error
-		sameMain bool
+		o             *Object
+		baseObj       fs.Object
+		currentXactID string
+		err           error
+		sameMain      bool
 	)
 
 	if f.useMeta {
@@ -856,7 +922,14 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		return nil, errors.Wrap(err, "can't detect composite file")
 	}
 
+	if f.useNoRename {
+		currentXactID, err = o.readXactID(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
 	caseInsensitive := f.features.CaseInsensitive
+
 	for _, dirOrObject := range entries {
 		entry, ok := dirOrObject.(fs.Object)
 		if !ok {
@@ -878,7 +951,7 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		if !sameMain {
 			continue // skip alien chunks
 		}
-		if ctrlType != "" || xactID != "" {
+		if ctrlType != "" || xactID != currentXactID {
 			if f.useMeta {
 				// temporary/control chunk calls for lazy metadata read
 				o.unsure = true
@@ -993,12 +1066,57 @@ func (o *Object) readMetadata(ctx context.Context) error {
 		}
 		o.md5 = metaInfo.md5
 		o.sha1 = metaInfo.sha1
+		o.xactID = metaInfo.xactID
 	}
 
 	o.isFull = true // cache results
+	o.xIDCached = true
 	return nil
 }
 
+// readXactID returns the transaction ID stored in the passed metadata object
+func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
+	// if xactID has already been read and cahced return it now
+	if o.xIDCached {
+		return o.xactID, nil
+	}
+	// Avoid reading metadata for backends that don't use xactID to identify permanent chunks
+	if !o.f.useNoRename {
+		return "", errors.New("readXactID requires norename transactions")
+	}
+	if o.main == nil {
+		return "", errors.New("readXactID requires valid metaobject")
+	}
+	if o.main.Size() > maxMetadataSize {
+		return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
+	}
+	reader, err := o.main.Open(ctx)
+	if err != nil {
+		return "", err
+	}
+	data, err := ioutil.ReadAll(reader)
+	_ = reader.Close() // ensure file handle is freed on windows
+	if err != nil {
+		return "", err
+	}
+
+	switch o.f.opt.MetaFormat {
+	case "simplejson":
+		if data != nil && len(data) > maxMetadataSizeWritten {
+			return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
+		}
+		var metadata metaSimpleJSON
+		err = json.Unmarshal(data, &metadata)
+		if err != nil {
+			return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
+		}
+		xactID = metadata.XactID
+	}
+	o.xactID = xactID
+	o.xIDCached = true
+	return xactID, nil
+}
+
 // put implements Put, PutStream, PutUnchecked, Update
 func (f *Fs) put(
 	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
@@ -1151,14 +1269,17 @@ func (f *Fs) put(
 	// If previous object was chunked, remove its chunks
 	f.removeOldChunks(ctx, baseRemote)
 
-	// Rename data chunks from temporary to final names
-	for chunkNo, chunk := range c.chunks {
-		chunkRemote := f.makeChunkName(baseRemote, chunkNo, "", "")
-		chunkMoved, errMove := f.baseMove(ctx, chunk, chunkRemote, delFailed)
-		if errMove != nil {
-			return nil, errMove
+	if !f.useNoRename {
+		// The transaction suffix will be removed for backends with quick rename operations
+		for chunkNo, chunk := range c.chunks {
+			chunkRemote := f.makeChunkName(baseRemote, chunkNo, "", "")
+			chunkMoved, errMove := f.baseMove(ctx, chunk, chunkRemote, delFailed)
+			if errMove != nil {
+				return nil, errMove
+			}
+			c.chunks[chunkNo] = chunkMoved
 		}
-		c.chunks[chunkNo] = chunkMoved
+		xactID = ""
 	}
 
 	if !f.useMeta {
@@ -1178,7 +1299,7 @@ func (f *Fs) put(
 	switch f.opt.MetaFormat {
 	case "simplejson":
 		c.updateHashes()
-		metadata, err = marshalSimpleJSON(ctx, sizeTotal, len(c.chunks), c.md5, c.sha1)
+		metadata, err = marshalSimpleJSON(ctx, sizeTotal, len(c.chunks), c.md5, c.sha1, xactID)
 	}
 	if err == nil {
 		metaInfo := f.wrapInfo(src, baseRemote, int64(len(metadata)))
@@ -1190,6 +1311,7 @@ func (f *Fs) put(
 
 	o := f.newObject("", metaObject, c.chunks)
 	o.size = sizeTotal
+	o.xactID = xactID
 	return o, nil
 }
 
@@ -1593,7 +1715,7 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 	var metadata []byte
 	switch f.opt.MetaFormat {
 	case "simplejson":
-		metadata, err = marshalSimpleJSON(ctx, newObj.size, len(newChunks), md5, sha1)
+		metadata, err = marshalSimpleJSON(ctx, newObj.size, len(newChunks), md5, sha1, o.xactID)
 		if err == nil {
 			metaInfo := f.wrapInfo(metaObject, "", int64(len(metadata)))
 			err = newObj.main.Update(ctx, bytes.NewReader(metadata), metaInfo)
@@ -1809,7 +1931,13 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		//fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
 		if entryType == fs.EntryObject {
 			mainPath, _, _, xactID := f.parseChunkName(path)
-			if mainPath != "" && xactID == "" {
+			metaXactID := ""
+			if f.useNoRename {
+				metaObject, _ := f.base.NewObject(ctx, mainPath)
+				dummyObject := f.newObject("", metaObject, nil)
+				metaXactID, _ = dummyObject.readXactID(ctx)
+			}
+			if mainPath != "" && xactID == metaXactID {
 				path = mainPath
 			}
 		}
@@ -1830,15 +1958,17 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // Object represents a composite file wrapping one or more data chunks
 type Object struct {
-	remote string
-	main   fs.Object   // meta object if file is composite, or wrapped non-chunked file, nil if meta format is 'none'
-	chunks []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
-	size   int64       // cached total size of chunks in a composite file or -1 for non-chunked files
-	isFull bool        // true if metadata has been read
-	unsure bool        // true if need to read metadata to detect object type
-	md5    string
-	sha1   string
-	f      *Fs
+	remote    string
+	main      fs.Object   // meta object if file is composite, or wrapped non-chunked file, nil if meta format is 'none'
+	chunks    []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
+	size      int64       // cached total size of chunks in a composite file or -1 for non-chunked files
+	isFull    bool        // true if metadata has been read
+	xIDCached bool        // true if xactID has been read
+	unsure    bool        // true if need to read metadata to detect object type
+	xactID    string      // transaction ID for "norename" or empty string for "renamed" chunks
+	md5       string
+	sha1      string
+	f         *Fs
 }
 
 func (o *Object) addChunk(chunk fs.Object, chunkNo int) error {
@@ -2166,6 +2296,7 @@ type ObjectInfo struct {
 	src     fs.ObjectInfo
 	fs      *Fs
 	nChunks int    // number of data chunks
+	xactID  string // transaction ID for "norename" or empty string for "renamed" chunks
 	size    int64  // overrides source size by the total size of data chunks
 	remote  string // overrides remote name
 	md5     string // overrides MD5 checksum
@@ -2264,8 +2395,9 @@ type metaSimpleJSON struct {
 	Size     *int64 `json:"size"`    // total size of data chunks
 	ChunkNum *int   `json:"nchunks"` // number of data chunks
 	// optional extra fields
-	MD5  string `json:"md5,omitempty"`
-	SHA1 string `json:"sha1,omitempty"`
+	MD5    string `json:"md5,omitempty"`
+	SHA1   string `json:"sha1,omitempty"`
+	XactID string `json:"txn,omitempty"` // transaction ID for norename transactions
 }
 
 // marshalSimpleJSON
@@ -2275,16 +2407,20 @@ type metaSimpleJSON struct {
 // - if file contents can be mistaken as meta object
 // - if consistent hashing is On but wrapped remote can't provide given hash
 //
-func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1 string) ([]byte, error) {
+func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) {
 	version := metadataVersion
+	if xactID == "" && version == 2 {
+		version = 1
+	}
 	metadata := metaSimpleJSON{
 		// required core fields
 		Version:  &version,
 		Size:     &size,
 		ChunkNum: &nChunks,
 		// optional extra fields
-		MD5:  md5,
-		SHA1: sha1,
+		MD5:    md5,
+		SHA1:   sha1,
+		XactID: xactID,
 	}
 	data, err := json.Marshal(&metadata)
 	if err == nil && data != nil && len(data) >= maxMetadataSizeWritten {
@@ -2362,6 +2498,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	info.nChunks = *metadata.ChunkNum
 	info.md5 = metadata.MD5
 	info.sha1 = metadata.SHA1
+	info.xactID = metadata.XactID
 	return info, true, nil
 }
 
@@ -2394,6 +2531,11 @@ func (f *Fs) Precision() time.Duration {
 	return f.base.Precision()
 }
 
+// CanQuickRename returns true if the Fs supports a quick rename operation
+func (f *Fs) CanQuickRename() bool {
+	return f.base.Features().Move != nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/chunker/chunker_internal_test.go

@@ -468,9 +468,15 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 		return obj
 	}
 	billyObj := newFile("billy")
+	billyTxn := billyObj.(*Object).xactID
+	if f.useNoRename {
+		require.True(t, billyTxn != "")
+	} else {
+		require.True(t, billyTxn == "")
+	}
 
 	billyChunkName := func(chunkNo int) string {
-		return f.makeChunkName(billyObj.Remote(), chunkNo, "", "")
+		return f.makeChunkName(billyObj.Remote(), chunkNo, "", billyTxn)
 	}
 
 	err := f.Mkdir(ctx, billyChunkName(1))
@@ -487,11 +493,13 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	// accessing chunks in strict mode is prohibited
 	f.opt.FailHard = true
 	billyChunk4Name := billyChunkName(4)
-	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
+	_, err = f.base.NewObject(ctx, billyChunk4Name)
+	require.NoError(t, err)
+	_, err = f.NewObject(ctx, billyChunk4Name)
 	assertOverlapError(err)
 
 	f.opt.FailHard = false
-	billyChunk4, err = f.NewObject(ctx, billyChunk4Name)
+	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
 	assert.NoError(t, err)
 	require.NotNil(t, billyChunk4)
 
@@ -520,7 +528,8 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 
 	// recreate billy in case it was anyhow corrupted
 	willyObj := newFile("willy")
-	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", "")
+	willyTxn := willyObj.(*Object).xactID
+	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", willyTxn)
 	f.opt.FailHard = false
 	willyChunk, err := f.NewObject(ctx, willyChunkName)
 	f.opt.FailHard = true
@@ -561,17 +570,20 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	contents := random.String(100)
 
-	newFile := func(f fs.Fs, name string) (fs.Object, string) {
-		filename := path.Join(dir, name)
+	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
+		filename = path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		_, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
-		return obj, filename
+		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
+			txnID = chunkObj.xactID
+		}
+		return
 	}
 
 	f.opt.FailHard = false
-	file, fileName := newFile(f, "wreaker")
-	wreak, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", ""))
+	file, fileName, fileTxn := newFile(f, "wreaker")
+	wreak, _, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", fileTxn))
 
 	f.opt.FailHard = false
 	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
@@ -650,7 +662,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		}
 	}
 
-	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "")
+	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "", "")
 	require.NoError(t, err)
 	todaysMeta := string(metaData)
 	runSubtest(todaysMeta, "today")
@@ -664,7 +676,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 	runSubtest(futureMeta, "future")
 }
 
-// test that chunker refuses to change on objects with future/unknowm metadata
+// Test that chunker refuses to change on objects with future/unknown metadata
 func testFutureProof(t *testing.T, f *Fs) {
 	if f.opt.MetaFormat == "none" {
 		t.Skip("this test requires metadata support")
@@ -738,6 +750,100 @@ func testFutureProof(t *testing.T, f *Fs) {
 	}
 }
 
+// The newer method of doing transactions without renaming should still be able to correctly process chunks that were created with renaming
+// If you attempt to do the inverse, however, the data chunks will be ignored causing commands to perform incorrectly
+func testBackwardsCompatibility(t *testing.T, f *Fs) {
+	if !f.useMeta {
+		t.Skip("Can't do norename transactions without metadata")
+	}
+	const dir = "backcomp"
+	ctx := context.Background()
+	saveOpt := f.opt
+	saveUseNoRename := f.useNoRename
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+		f.useNoRename = saveUseNoRename
+	}()
+	f.opt.ChunkSize = fs.SizeSuffix(10)
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	contents := random.String(250)
+	newFile := func(f fs.Fs, name string) (fs.Object, string) {
+		filename := path.Join(dir, name)
+		item := fstest.Item{Path: filename, ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		require.NotNil(t, obj)
+		return obj, filename
+	}
+
+	f.opt.FailHard = false
+	f.useNoRename = false
+	file, fileName := newFile(f, "renamefile")
+
+	f.opt.FailHard = false
+	item := fstest.NewItem(fileName, contents, modTime)
+
+	var items []fstest.Item
+	items = append(items, item)
+
+	f.useNoRename = true
+	fstest.CheckListingWithRoot(t, f, dir, items, nil, f.Precision())
+	_, err := f.NewObject(ctx, fileName)
+	assert.NoError(t, err)
+
+	f.opt.FailHard = true
+	_, err = f.List(ctx, dir)
+	assert.NoError(t, err)
+
+	f.opt.FailHard = false
+	_ = file.Remove(ctx)
+}
+
+func testChunkerServerSideMove(t *testing.T, f *Fs) {
+	if !f.useMeta {
+		t.Skip("Can't test norename transactions without metadata")
+	}
+
+	ctx := context.Background()
+	const dir = "servermovetest"
+	subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), dir)
+
+	subFs1, err := fs.NewFs(ctx, subRemote+"/subdir1")
+	assert.NoError(t, err)
+	fs1, isChunkerFs := subFs1.(*Fs)
+	assert.True(t, isChunkerFs)
+	fs1.useNoRename = false
+	fs1.opt.ChunkSize = fs.SizeSuffix(3)
+
+	subFs2, err := fs.NewFs(ctx, subRemote+"/subdir2")
+	assert.NoError(t, err)
+	fs2, isChunkerFs := subFs2.(*Fs)
+	assert.True(t, isChunkerFs)
+	fs2.useNoRename = true
+	fs2.opt.ChunkSize = fs.SizeSuffix(3)
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	item := fstest.Item{Path: "movefile", ModTime: modTime}
+	contents := "abcdef"
+	_, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
+
+	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
+	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
+	assert.NoError(t, err)
+	assert.Equal(t, int64(len(contents)), dstFile.Size())
+
+	r, err := dstFile.Open(ctx)
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, contents, string(data))
+	_ = r.Close()
+	_ = operations.Purge(ctx, f.base, dir)
+}
+
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("PutLarge", func(t *testing.T) {
@@ -764,6 +870,12 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("FutureProof", func(t *testing.T) {
 		testFutureProof(t, f)
 	})
+	t.Run("BackwardsCompatibility", func(t *testing.T) {
+		testBackwardsCompatibility(t, f)
+	})
+	t.Run("ChunkerServerSideMove", func(t *testing.T) {
+		testChunkerServerSideMove(t, f)
+	})
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/crypt/crypt.go

@@ -101,6 +101,21 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
+		}, {
+			Name:     "no_data_encryption",
+			Help:     "Option to either encrypt file data or leave it unencrypted.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{
+				{
+					Value: "true",
+					Help:  "Don't encrypt file data, leave it unencrypted.",
+				},
+				{
+					Value: "false",
+					Help:  "Encrypt file data.",
+				},
+			},
 		}},
 	})
 }
@@ -209,6 +224,7 @@ type Options struct {
 	Remote                  string `config:"remote"`
 	FilenameEncryption      string `config:"filename_encryption"`
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
+	NoDataEncryption        bool   `config:"no_data_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
@@ -346,6 +362,10 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	if f.opt.NoDataEncryption {
+		return put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
+	}
+
 	// Encrypt the data into wrappedIn
 	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
@@ -617,6 +637,10 @@ func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Objec
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+	if f.opt.NoDataEncryption {
+		return src.Hash(ctx, hashType)
+	}
+
 	// Read the nonce - opening the file is sufficient to read the nonce in
 	// use a limited read so we only read the header
 	in, err := o.Object.Open(ctx, &fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
@@ -822,9 +846,13 @@ func (o *Object) Remote() string {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	size, err := o.f.cipher.DecryptedSize(o.Object.Size())
-	if err != nil {
-		fs.Debugf(o, "Bad size for decrypt: %v", err)
+	size := o.Object.Size()
+	if !o.f.opt.NoDataEncryption {
+		var err error
+		size, err = o.f.cipher.DecryptedSize(size)
+		if err != nil {
+			fs.Debugf(o, "Bad size for decrypt: %v", err)
+		}
 	}
 	return size
 }
@@ -842,6 +870,10 @@ func (o *Object) UnWrap() fs.Object {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	if o.f.opt.NoDataEncryption {
+		return o.Object.Open(ctx, options...)
+	}
+
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
 	for _, option := range options {

backend/crypt/crypt_test.go

@@ -91,3 +91,26 @@ func TestObfuscate(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
+
+// TestNoDataObfuscate runs integration tests against the remote
+func TestNoDataObfuscate(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
+	name := "TestCrypt4"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
+			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
+			{Name: name, Key: "no_data_encryption", Value: "true"},
+		},
+		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}

backend/drive/drive.go

@@ -207,7 +207,7 @@ func init() {
 			}
 			err = configTeamDrive(ctx, opt, m, name)
 			if err != nil {
-				log.Fatalf("Failed to configure team drive: %v", err)
+				log.Fatalf("Failed to configure Shared Drive: %v", err)
 			}
 		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
@@ -247,7 +247,7 @@ a non root folder as its starting point.
 			Advanced: true,
 		}, {
 			Name:     "team_drive",
-			Help:     "ID of the Team Drive",
+			Help:     "ID of the Shared Drive (Team Drive)",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
 		}, {
@@ -590,13 +590,13 @@ type Fs struct {
 }
 
 type baseObject struct {
-	fs           *Fs    // what this object is part of
-	remote       string // The remote path
-	id           string // Drive Id of this object
-	modifiedDate string // RFC3339 time it was last modified
-	mimeType     string // The object MIME type
-	bytes        int64  // size of the object
-	parents      int    // number of parents
+	fs           *Fs      // what this object is part of
+	remote       string   // The remote path
+	id           string   // Drive Id of this object
+	modifiedDate string   // RFC3339 time it was last modified
+	mimeType     string   // The object MIME type
+	bytes        int64    // size of the object
+	parents      []string // IDs of the parent directories
 }
 type documentObject struct {
 	baseObject
@@ -666,7 +666,7 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
-				fs.Errorf(f, "Received team drive file limit error: %v", err)
+				fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			}
 		}
@@ -955,24 +955,24 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 		return nil
 	}
 	if opt.TeamDriveID == "" {
-		fmt.Printf("Configure this as a team drive?\n")
+		fmt.Printf("Configure this as a Shared Drive (Team Drive)?\n")
 	} else {
-		fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
+		fmt.Printf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID)
 	}
 	if !config.Confirm(false) {
 		return nil
 	}
 	f, err := newFs(ctx, name, "", m)
 	if err != nil {
-		return errors.Wrap(err, "failed to make Fs to list teamdrives")
+		return errors.Wrap(err, "failed to make Fs to list Shared Drives")
 	}
-	fmt.Printf("Fetching team drive list...\n")
+	fmt.Printf("Fetching Shared Drive list...\n")
 	teamDrives, err := f.listTeamDrives(ctx)
 	if err != nil {
 		return err
 	}
 	if len(teamDrives) == 0 {
-		fmt.Printf("No team drives found in your account")
+		fmt.Printf("No Shared Drives found in your account")
 		return nil
 	}
 	var driveIDs, driveNames []string
@@ -980,7 +980,7 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 		driveIDs = append(driveIDs, teamDrive.Id)
 		driveNames = append(driveNames, teamDrive.Name)
 	}
-	driveID := config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true)
+	driveID := config.Choose("Enter a Shared Drive ID", driveIDs, driveNames, true)
 	m.Set("team_drive", driveID)
 	m.Set("root_folder_id", "")
 	opt.TeamDriveID = driveID
@@ -1236,7 +1236,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 		modifiedDate: modifiedDate,
 		mimeType:     info.MimeType,
 		bytes:        size,
-		parents:      len(info.Parents),
+		parents:      info.Parents,
 	}
 }
 
@@ -2019,6 +2019,9 @@ func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry,
 		f.dirCache.Put(remote, item.Id)
 		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
 		d := fs.NewDir(remote, when).SetID(item.Id)
+		if len(item.Parents) > 0 {
+			d.SetParentID(item.Parents[0])
+		}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
@@ -2475,9 +2478,9 @@ func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
 		return f.shouldRetry(err)
 	})
 	if err != nil {
-		return errors.Wrap(err, "failed to get Team/Shared Drive info")
+		return errors.Wrap(err, "failed to get Shared Drive info")
 	}
-	fs.Debugf(f, "read info from team drive %q", td.Name)
+	fs.Debugf(f, "read info from Shared Drive %q", td.Name)
 	return err
 }
 
@@ -2963,7 +2966,7 @@ func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err
 			return defaultFs.shouldRetry(err)
 		})
 		if err != nil {
-			return drives, errors.Wrap(err, "listing team drives failed")
+			return drives, errors.Wrap(err, "listing Team Drives failed")
 		}
 		drives = append(drives, teamDrives.TeamDrives...)
 		if teamDrives.NextPageToken == "" {
@@ -3131,8 +3134,8 @@ authenticated with "drive2:" can't read files from "drive:".
 	},
 }, {
 	Name:  "drives",
-	Short: "List the shared drives available to this account",
-	Long: `This command lists the shared drives (teamdrives) available to this
+	Short: "List the Shared Drives available to this account",
+	Long: `This command lists the Shared Drives (Team Drives) available to this
 account.
 
 Usage:
@@ -3722,7 +3725,7 @@ func (o *linkObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo
 
 // Remove an object
 func (o *baseObject) Remove(ctx context.Context) error {
-	if o.parents > 1 {
+	if len(o.parents) > 1 {
 		return errors.New("can't delete safely - has multiple parents")
 	}
 	return o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
@@ -3738,6 +3741,14 @@ func (o *baseObject) ID() string {
 	return o.id
 }
 
+// ParentID returns the ID of the Object parent if known, or "" if not
+func (o *baseObject) ParentID() string {
+	if len(o.parents) > 0 {
+		return o.parents[0]
+	}
+	return ""
+}
+
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
@@ -3798,10 +3809,13 @@ var (
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
+	_ fs.ParentIDer      = (*Object)(nil)
 	_ fs.Object          = (*documentObject)(nil)
 	_ fs.MimeTyper       = (*documentObject)(nil)
 	_ fs.IDer            = (*documentObject)(nil)
+	_ fs.ParentIDer      = (*documentObject)(nil)
 	_ fs.Object          = (*linkObject)(nil)
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
+	_ fs.ParentIDer      = (*linkObject)(nil)
 )

backend/dropbox/dropbox.go

@@ -94,7 +94,14 @@ const (
 var (
 	// Description of how to auth for this app
 	dropboxConfig = &oauth2.Config{
-		Scopes: []string{},
+		Scopes: []string{
+			"files.metadata.write",
+			"files.content.write",
+			"files.content.read",
+			"sharing.write",
+			// "file_requests.write",
+			// "members.read", // needed for impersonate - but causes app to need to be approved by Dropbox Team Admin during the flow
+		},
 		// Endpoint: oauth2.Endpoint{
 		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
 		// 	TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
@@ -115,6 +122,19 @@ var (
 	errNotSupportedInSharedMode = fserrors.NoRetryError(errors.New("not supported in shared files mode"))
 )
 
+// Gets an oauth config with the right scopes
+func getOauthConfig(m configmap.Mapper) *oauth2.Config {
+	// If not impersonating, use standard scopes
+	if impersonate, _ := m.Get("impersonate"); impersonate == "" {
+		return dropboxConfig
+	}
+	// Make a copy of the config
+	config := *dropboxConfig
+	// Make a copy of the scopes with "members.read" appended
+	config.Scopes = append(config.Scopes, "members.read")
+	return &config
+}
+
 // Register with Fs
 func init() {
 	DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New)
@@ -129,7 +149,7 @@ func init() {
 					oauth2.SetAuthURLParam("token_access_type", "offline"),
 				},
 			}
-			err := oauthutil.Config(ctx, "dropbox", name, m, dropboxConfig, &opt)
+			err := oauthutil.Config(ctx, "dropbox", name, m, getOauthConfig(m), &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -147,8 +167,23 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
-			Name:     "impersonate",
-			Help:     "Impersonate this user when using a business account.",
+			Name: "impersonate",
+			Help: `Impersonate this user when using a business account.
+
+Note that if you want to use impersonate, you should make sure this
+flag is set when running "rclone config" as this will cause rclone to
+request the "members.read" scope which it won't normally. This is
+needed to lookup a members email address into the internal ID that
+dropbox uses in the API.
+
+Using the "members.read" scope will require a Dropbox Team Admin
+to approve during the OAuth flow.
+
+You will have to use your own App (setting your own client_id and
+client_secret) to use this option as currently rclone's default set of
+permissions doesn't include "members.read". This can be added once
+v1.55 or later is in use everywhere.
+`,
 			Default:  "",
 			Advanced: true,
 		}, {
@@ -184,11 +219,11 @@ shared folder.`,
 			// as invalid characters.
 			// Testing revealed names with trailing spaces and the DEL character don't work.
 			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
-			Default: (encoder.Base |
+			Default: encoder.Base |
 				encoder.EncodeBackSlash |
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
-				encoder.EncodeInvalidUtf8),
+				encoder.EncodeInvalidUtf8,
 		}}...),
 	})
 }
@@ -207,8 +242,10 @@ type Fs struct {
 	name           string         // name of this remote
 	root           string         // the path we are working on
 	opt            Options        // parsed options
+	ci             *fs.ConfigInfo // global config
 	features       *fs.Features   // optional features
 	srv            files.Client   // the connection to the dropbox server
+	svc            files.Client   // the connection to the dropbox server (unauthorized)
 	sharing        sharing.Client // as above, but for generating sharing links
 	users          users.Client   // as above, but for accessing user information
 	team           team.Client    // for the Teams API
@@ -255,7 +292,10 @@ func (f *Fs) Features() *fs.Features {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err == nil {
 		return false, err
 	}
@@ -327,27 +367,34 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	}
 
-	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, dropboxConfig)
+	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, getOauthConfig(m))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure dropbox")
 	}
 
+	ci := fs.GetConfig(ctx)
+
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
+		ci:    ci,
 		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
-	config := dropbox.Config{
+	cfg := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
 		Client:          oAuthClient,    // maybe???
 		HeaderGenerator: f.headerGenerator,
 	}
 
+	// unauthorized config for endpoints that fail with auth
+	ucfg := dropbox.Config{
+		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
+	}
+
 	// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
-	f.team = team.New(config)
+	f.team = team.New(cfg)
 
 	if opt.Impersonate != "" {
-
 		user := team.UserSelectorArg{
 			Email: opt.Impersonate,
 		}
@@ -362,12 +409,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, errors.Wrapf(err, "invalid dropbox team member: %q", opt.Impersonate)
 		}
 
-		config.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
+		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
 
-	f.srv = files.New(config)
-	f.sharing = sharing.New(config)
-	f.users = users.New(config)
+	f.srv = files.New(cfg)
+	f.svc = files.New(ucfg)
+	f.sharing = sharing.New(cfg)
+	f.users = users.New(cfg)
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
 		ReadMimeType:            false,
@@ -380,7 +428,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if f.root == "" {
 			return f, nil
 		}
-		_, err := f.findSharedFile(f.root)
+		_, err := f.findSharedFile(ctx, f.root)
 		f.root = ""
 		if err == nil {
 			return f, fs.ErrorIsFile
@@ -400,7 +448,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 
 		// root is not empty so we have find the right shared folder if it exists
-		id, err := f.findSharedFolder(dir)
+		id, err := f.findSharedFolder(ctx, dir)
 		if err != nil {
 			// if we didn't find the specified shared folder we have to bail out here
 			return nil, err
@@ -408,7 +456,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// we found the specified shared folder so let's mount it
 		// this will add it to the users normal root namespace and allows us
 		// to actually perform operations on it using the normal api endpoints.
-		err = f.mountSharedFolder(id)
+		err = f.mountSharedFolder(ctx, id)
 		if err != nil {
 			switch e := err.(type) {
 			case sharing.MountFolderAPIError:
@@ -432,7 +480,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
 			acc, err = f.users.GetCurrentAccount()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "get current account failed")
@@ -450,7 +498,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.setRoot(root)
 
 	// See if the root is actually an object
-	_, err = f.getFileMetadata(f.slashRoot)
+	_, err = f.getFileMetadata(ctx, f.slashRoot)
 	if err == nil {
 		newRoot := path.Dir(f.root)
 		if newRoot == "." {
@@ -484,12 +532,12 @@ func (f *Fs) setRoot(root string) {
 }
 
 // getMetadata gets the metadata for a file or directory
-func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
+func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		switch e := err.(type) {
@@ -504,8 +552,8 @@ func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool,
 }
 
 // getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
-	entry, notFound, err := f.getMetadata(filePath)
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
+	entry, notFound, err := f.getMetadata(ctx, filePath)
 	if err != nil {
 		return nil, err
 	}
@@ -520,8 +568,8 @@ func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err
 }
 
 // getDirMetadata gets the metadata for a directory
-func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
-	entry, notFound, err := f.getMetadata(dirPath)
+func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
+	entry, notFound, err := f.getMetadata(ctx, dirPath)
 	if err != nil {
 		return nil, err
 	}
@@ -538,7 +586,7 @@ func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -547,7 +595,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
 	if info != nil {
 		err = o.setMetadataFromEntry(info)
 	} else {
-		err = o.readEntryAndSetMetadata()
+		err = o.readEntryAndSetMetadata(ctx)
 	}
 	if err != nil {
 		return nil, err
@@ -559,14 +607,14 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if f.opt.SharedFiles {
-		return f.findSharedFile(remote)
+		return f.findSharedFile(ctx, remote)
 	}
-	return f.newObjectWithInfo(remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
 // listSharedFoldersApi lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
-func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
+func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
 	var res *sharing.ListFoldersResult
 	for {
@@ -576,7 +624,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListFolders(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, err
@@ -588,7 +636,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListFoldersContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -613,8 +661,8 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 // findSharedFolder find the id for a given shared folder name
 // somewhat annoyingly there is no endpoint to query a shared folder by it's name
 // so our only option is to iterate over all shared folders
-func (f *Fs) findSharedFolder(name string) (id string, err error) {
-	entries, err := f.listSharedFolders()
+func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
+	entries, err := f.listSharedFolders(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -626,21 +674,21 @@ func (f *Fs) findSharedFolder(name string) (id string, err error) {
 	return "", fs.ErrorDirNotFound
 }
 
-// mountSharedFolders mount a shared folder to the root namespace
-func (f *Fs) mountSharedFolder(id string) error {
+// mountSharedFolder mount a shared folder to the root namespace
+func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
 	arg := sharing.MountFolderArg{
 		SharedFolderId: id,
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.sharing.MountFolder(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
 
-// listSharedFolders lists shared the user as access to (note this means individual
+// listReceivedFiles lists shared the user as access to (note this means individual
 // files not files contained in shared folders)
-func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
+func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
 	var res *sharing.ListFilesResult
 	for {
@@ -650,7 +698,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListReceivedFiles(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, err
@@ -662,7 +710,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListReceivedFilesContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -689,8 +737,8 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-func (f *Fs) findSharedFile(name string) (o *Object, err error) {
-	files, err := f.listReceivedFiles()
+func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
+	files, err := f.listReceivedFiles(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -713,10 +761,10 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) {
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	if f.opt.SharedFiles {
-		return f.listReceivedFiles()
+		return f.listReceivedFiles(ctx)
 	}
 	if f.opt.SharedFolders {
-		return f.listSharedFolders()
+		return f.listSharedFolders(ctx)
 	}
 
 	root := f.slashRoot
@@ -737,7 +785,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			err = f.pacer.Call(func() (bool, error) {
 				res, err = f.srv.ListFolder(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				switch e := err.(type) {
@@ -755,7 +803,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			err = f.pacer.Call(func() (bool, error) {
 				res, err = f.srv.ListFolderContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -785,7 +833,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
 				entries = append(entries, d)
 			} else if fileInfo != nil {
-				o, err := f.newObjectWithInfo(remote, fileInfo)
+				o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
 				if err != nil {
 					return nil, err
 				}
@@ -834,7 +882,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	}
 
 	// check directory doesn't exist
-	_, err := f.getDirMetadata(root)
+	_, err := f.getDirMetadata(ctx, root)
 	if err == nil {
 		return nil // directory exists already
 	} else if err != fs.ErrorDirNotFound {
@@ -851,7 +899,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.CreateFolderV2(&arg2)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
@@ -868,7 +916,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	if check {
 		// check directory exists
-		_, err = f.getDirMetadata(root)
+		_, err = f.getDirMetadata(ctx, root)
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
 		}
@@ -885,7 +933,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
 			res, err = f.srv.ListFolder(&arg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
@@ -898,7 +946,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
@@ -951,7 +999,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.CopyV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "copy failed")
@@ -1012,7 +1060,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.MoveV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
@@ -1046,7 +1094,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
 		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 
 	if err != nil && strings.Contains(err.Error(),
@@ -1059,7 +1107,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		var listRes *sharing.ListSharedLinksResult
 		err = f.pacer.Call(func() (bool, error) {
 			listRes, err = f.sharing.ListSharedLinks(&listArg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return
@@ -1101,7 +1149,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	dstPath := path.Join(f.slashRoot, dstRemote)
 
 	// Check if destination exists
-	_, err := f.getDirMetadata(dstPath)
+	_, err := f.getDirMetadata(ctx, dstPath)
 	if err == nil {
 		return fs.ErrorDirExists
 	} else if err != fs.ErrorDirNotFound {
@@ -1120,7 +1168,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.MoveV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "MoveDir failed")
@@ -1134,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var q *users.SpaceUsage
 	err = f.pacer.Call(func() (bool, error) {
 		q, err = f.users.GetSpaceUsage()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -1156,6 +1204,159 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	return usage, nil
 }
 
+// ChangeNotify calls the passed function with a path that has had changes.
+// If the implementation uses polling, it should adhere to the given interval.
+//
+// Automatically restarts itself in case of unexpected behavior of the remote.
+//
+// Close the returned channel to stop being notified.
+func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+	go func() {
+		// get the StartCursor early so all changes from now on get processed
+		startCursor, err := f.changeNotifyCursor(ctx)
+		if err != nil {
+			fs.Infof(f, "Failed to get StartCursor: %s", err)
+		}
+		var ticker *time.Ticker
+		var tickerC <-chan time.Time
+		for {
+			select {
+			case pollInterval, ok := <-pollIntervalChan:
+				if !ok {
+					if ticker != nil {
+						ticker.Stop()
+					}
+					return
+				}
+				if ticker != nil {
+					ticker.Stop()
+					ticker, tickerC = nil, nil
+				}
+				if pollInterval != 0 {
+					ticker = time.NewTicker(pollInterval)
+					tickerC = ticker.C
+				}
+			case <-tickerC:
+				if startCursor == "" {
+					startCursor, err = f.changeNotifyCursor(ctx)
+					if err != nil {
+						fs.Infof(f, "Failed to get StartCursor: %s", err)
+						continue
+					}
+				}
+				fs.Debugf(f, "Checking for changes on remote")
+				startCursor, err = f.changeNotifyRunner(ctx, notifyFunc, startCursor)
+				if err != nil {
+					fs.Infof(f, "Change notify listener failure: %s", err)
+				}
+			}
+		}
+	}()
+}
+
+func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) {
+	var startCursor *files.ListFolderGetLatestCursorResult
+
+	err = f.pacer.Call(func() (bool, error) {
+		arg := files.ListFolderArg{
+			Path:      f.opt.Enc.FromStandardPath(f.slashRoot),
+			Recursive: true,
+		}
+
+		if arg.Path == "/" {
+			arg.Path = ""
+		}
+
+		startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
+
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return
+	}
+	return startCursor.Cursor, nil
+}
+
+func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
+	cursor := startCursor
+	var res *files.ListFolderLongpollResult
+
+	// Dropbox sets a timeout range of 30 - 480
+	timeout := uint64(f.ci.TimeoutOrInfinite() / time.Second)
+
+	if timeout < 30 {
+		timeout = 30
+	}
+
+	if timeout > 480 {
+		timeout = 480
+	}
+
+	err = f.pacer.Call(func() (bool, error) {
+		args := files.ListFolderLongpollArg{
+			Cursor:  cursor,
+			Timeout: timeout,
+		}
+
+		res, err = f.svc.ListFolderLongpoll(&args)
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return
+	}
+
+	if !res.Changes {
+		return cursor, nil
+	}
+
+	if res.Backoff != 0 {
+		fs.Debugf(f, "Waiting to poll for %d seconds", res.Backoff)
+		time.Sleep(time.Duration(res.Backoff) * time.Second)
+	}
+
+	for {
+		var changeList *files.ListFolderResult
+
+		arg := files.ListFolderContinueArg{
+			Cursor: cursor,
+		}
+		err = f.pacer.Call(func() (bool, error) {
+			changeList, err = f.srv.ListFolderContinue(&arg)
+			return shouldRetry(ctx, err)
+		})
+		if err != nil {
+			return "", errors.Wrap(err, "list continue")
+		}
+		cursor = changeList.Cursor
+		var entryType fs.EntryType
+		for _, entry := range changeList.Entries {
+			entryPath := ""
+			switch info := entry.(type) {
+			case *files.FolderMetadata:
+				entryType = fs.EntryDirectory
+				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+			case *files.FileMetadata:
+				entryType = fs.EntryObject
+				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+			case *files.DeletedMetadata:
+				entryType = fs.EntryObject
+				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+			default:
+				fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
+				continue
+			}
+
+			if entryPath != "" {
+				notifyFunc(entryPath, entryType)
+			}
+		}
+		if !changeList.HasMore {
+			break
+		}
+	}
+	return cursor, nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(DbHashType)
@@ -1194,7 +1395,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != DbHashType {
 		return "", hash.ErrUnsupported
 	}
-	err := o.readMetaData()
+	err := o.readMetaData(ctx)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to read hash from metadata")
 	}
@@ -1218,17 +1419,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
 }
 
 // Reads the entry for a file from dropbox
-func (o *Object) readEntry() (*files.FileMetadata, error) {
-	return o.fs.getFileMetadata(o.remotePath())
+func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
+	return o.fs.getFileMetadata(ctx, o.remotePath())
 }
 
 // Read entry if not set and set metadata from it
-func (o *Object) readEntryAndSetMetadata() error {
+func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
 	// Last resort set time from client
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	entry, err := o.readEntry()
+	entry, err := o.readEntry(ctx)
 	if err != nil {
 		return err
 	}
@@ -1241,12 +1442,12 @@ func (o *Object) remotePath() string {
 }
 
 // readMetaData gets the info if it hasn't already been fetched
-func (o *Object) readMetaData() (err error) {
+func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if !o.modTime.IsZero() {
 		return nil
 	}
 	// Last resort
-	return o.readEntryAndSetMetadata()
+	return o.readEntryAndSetMetadata(ctx)
 }
 
 // ModTime returns the modification time of the object
@@ -1254,7 +1455,7 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
-	err := o.readMetaData()
+	err := o.readMetaData(ctx)
 	if err != nil {
 		fs.Debugf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1288,7 +1489,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		err = o.fs.pacer.Call(func() (bool, error) {
 			_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -1304,7 +1505,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, in, err = o.fs.srv.Download(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 
 	switch e := err.(type) {
@@ -1323,7 +1524,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // Will work optimally if size is >= uploadChunkSize. If the size is either
 // unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
 // avoidable request to the Dropbox API that does not carry payload.
-func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
+func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
 	chunkSize := int64(o.fs.opt.ChunkSize)
 	chunks := 0
 	if size != -1 {
@@ -1352,7 +1553,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 			return false, nil
 		}
 		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1478,11 +1679,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var err error
 	var entry *files.FileMetadata
 	if size > int64(o.fs.opt.ChunkSize) || size == -1 {
-		entry, err = o.uploadChunked(in, commitInfo, size)
+		entry, err = o.uploadChunked(ctx, in, commitInfo, size)
 	} else {
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			entry, err = o.fs.srv.Upload(commitInfo, in)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	}
 	if err != nil {
@@ -1500,7 +1701,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
 			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
 		})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }

backend/fichier/api.go

@@ -28,7 +28,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// Detect this error which the integration tests provoke
 	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
 	//
@@ -48,6 +51,41 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 
 var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
 
+func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) {
+	// Create the directory for the object if it doesn't exist
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return
+	}
+	// Temporary Object under construction
+	o = &Object{
+		fs:     f,
+		remote: remote,
+	}
+	return o, leaf, directoryID, nil
+}
+
+func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
+	request := FileInfoRequest{
+		URL: url,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/info.cgi",
+	}
+
+	var file File
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't read file info")
+	}
+
+	return &file, err
+}
+
 func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
 	request := DownloadRequest{
 		URL:    url,
@@ -61,7 +99,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
 	var token GetTokenResponse
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -89,7 +127,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
 	var sharedFiles SharedFolderResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -118,7 +156,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
 	filesList = &FilesList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -146,7 +184,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 	foldersList = &FoldersList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list folders")
@@ -240,7 +278,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons
 	response = &MakeFolderResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create folder")
@@ -267,7 +305,7 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't remove folder")
@@ -296,7 +334,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
 	response = &GenericOKResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -308,6 +346,56 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
 	return response, nil
 }
 
+func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename string) (response *MoveFileResponse, err error) {
+	request := &MoveFileRequest{
+		URLs:     []string{url},
+		FolderID: folderID,
+		Rename:   rename,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/mv.cgi",
+	}
+
+	response = &MoveFileResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
+		return shouldRetry(ctx, resp, err)
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't copy file")
+	}
+
+	return response, nil
+}
+
+func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
+	request := &CopyFileRequest{
+		URLs:     []string{url},
+		FolderID: folderID,
+		Rename:   rename,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/cp.cgi",
+	}
+
+	response = &CopyFileResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
+		return shouldRetry(ctx, resp, err)
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't copy file")
+	}
+
+	return response, nil
+}
+
 func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
 	// fs.Debugf(f, "Requesting Upload node")
 
@@ -320,7 +408,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 	response = &GetUploadNodeResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "didnt got an upload node")
@@ -363,7 +451,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -397,7 +485,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
 	response = &EndFileUploadResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {

backend/fichier/fichier.go

@@ -363,7 +363,6 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		fs:     f,
 		remote: remote,
 		file: File{
-			ACL:         0,
 			CDN:         0,
 			Checksum:    link.Whirlpool,
 			ContentType: "",
@@ -416,9 +415,89 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return nil
 }
 
+// Move src to this remote using server side move operations.
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Create temporary object
+	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
+	if err != nil {
+		return nil, err
+	}
+
+	folderID, err := strconv.Atoi(directoryID)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't move file")
+	}
+	if resp.Status != "OK" {
+		return nil, errors.New("couldn't move file")
+	}
+
+	file, err := f.readFileInfo(ctx, resp.URLs[0])
+	if err != nil {
+		return nil, errors.New("couldn't read file data")
+	}
+	dstObj.setMetaData(*file)
+	return dstObj, nil
+}
+
+// Copy src to this remote using server side move operations.
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Create temporary object
+	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
+	if err != nil {
+		return nil, err
+	}
+
+	folderID, err := strconv.Atoi(directoryID)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't move file")
+	}
+	if resp.Status != "OK" {
+		return nil, errors.New("couldn't move file")
+	}
+
+	file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL)
+	if err != nil {
+		return nil, errors.New("couldn't read file data")
+	}
+	dstObj.setMetaData(*file)
+	return dstObj, nil
+}
+
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	o, err := f.NewObject(ctx, remote)
+	if err != nil {
+		return "", err
+	}
+	return o.(*Object).file.URL, nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ dircache.DirCacher = (*Fs)(nil)
 )

backend/fichier/object.go

@@ -72,6 +72,10 @@ func (o *Object) SetModTime(context.Context, time.Time) error {
 	//return errors.New("setting modtime is not supported for 1fichier remotes")
 }
 
+func (o *Object) setMetaData(file File) {
+	o.file = file
+}
+
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
 	fs.FixRangeOption(options, o.file.Size)
@@ -90,7 +94,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.rest.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {

backend/fichier/structs.go

@@ -1,5 +1,10 @@
 package fichier
 
+// FileInfoRequest is the request structure of the corresponding request
+type FileInfoRequest struct {
+	URL string `json:"url"`
+}
+
 // ListFolderRequest is the request structure of the corresponding request
 type ListFolderRequest struct {
 	FolderID int `json:"folder_id"`
@@ -49,6 +54,39 @@ type MakeFolderResponse struct {
 	FolderID int    `json:"folder_id"`
 }
 
+// MoveFileRequest is the request structure of the corresponding request
+type MoveFileRequest struct {
+	URLs     []string `json:"urls"`
+	FolderID int      `json:"destination_folder_id"`
+	Rename   string   `json:"rename,omitempty"`
+}
+
+// MoveFileResponse is the response structure of the corresponding request
+type MoveFileResponse struct {
+	Status string   `json:"status"`
+	URLs   []string `json:"urls"`
+}
+
+// CopyFileRequest is the request structure of the corresponding request
+type CopyFileRequest struct {
+	URLs     []string `json:"urls"`
+	FolderID int      `json:"folder_id"`
+	Rename   string   `json:"rename,omitempty"`
+}
+
+// CopyFileResponse is the response structure of the corresponding request
+type CopyFileResponse struct {
+	Status string     `json:"status"`
+	Copied int        `json:"copied"`
+	URLs   []FileCopy `json:"urls"`
+}
+
+// FileCopy is used in the the CopyFileResponse
+type FileCopy struct {
+	FromURL string `json:"from_url"`
+	ToURL   string `json:"to_url"`
+}
+
 // GetUploadNodeResponse is the response structure of the corresponding request
 type GetUploadNodeResponse struct {
 	ID  string `json:"id"`
@@ -86,7 +124,6 @@ type EndFileUploadResponse struct {
 
 // File is the structure how 1Fichier returns a File
 type File struct {
-	ACL         int    `json:"acl"`
 	CDN         int    `json:"cdn"`
 	Checksum    string `json:"checksum"`
 	ContentType string `json:"content-type"`

backend/filefabric/filefabric.go

@@ -228,7 +228,10 @@ var retryStatusCodes = []struct {
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err != nil {
 		return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 	}
@@ -401,7 +404,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr
 		// Refresh the body each retry
 		opts.Body = strings.NewReader(data.Encode())
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, result)
-		return f.shouldRetry(resp, err, result)
+		return f.shouldRetry(ctx, resp, err, result)
 	})
 	if err != nil {
 		return resp, err
@@ -1277,7 +1280,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader)
-		return o.fs.shouldRetry(resp, err, nil)
+		return o.fs.shouldRetry(ctx, resp, err, nil)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to upload")

backend/ftp/ftp.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"io"
+	"net"
 	"net/textproto"
 	"path"
 	"runtime"
@@ -20,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
@@ -91,6 +93,17 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:    "idle_timeout",
+			Default: fs.Duration(60 * time.Second),
+			Help: `Max time before closing idle connections
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
+`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -118,6 +131,7 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
+	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -134,7 +148,9 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
+	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
+	tlsConf  *tls.Config
 }
 
 // Object describes an FTP file
@@ -211,25 +227,36 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+type dialCtx struct {
+	f   *Fs
+	ctx context.Context
+}
+
+// dial a new connection with fshttp dialer
+func (d *dialCtx) dial(network, address string) (net.Conn, error) {
+	conn, err := fshttp.NewDialer(d.ctx).Dial(network, address)
+	if err != nil {
+		return nil, err
+	}
+	if d.f.tlsConf != nil {
+		conn = tls.Client(conn, d.f.tlsConf)
+	}
+	return conn, err
+}
+
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
-	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(f.ci.ConnectTimeout)}
-	if f.opt.TLS && f.opt.ExplicitTLS {
-		fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
-		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
-	} else if f.opt.TLS {
-		tlsConfig := &tls.Config{
-			ServerName:         f.opt.Host,
-			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+	dCtx := dialCtx{f, ctx}
+	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dCtx.dial)}
+	if f.opt.ExplicitTLS {
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
+		// Initial connection needs to be cleartext for explicit TLS
+		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
+		if err != nil {
+			return nil, err
 		}
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
-	} else if f.opt.ExplicitTLS {
-		tlsConfig := &tls.Config{
-			ServerName:         f.opt.Host,
-			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
+		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -308,9 +335,32 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+	}
 	f.poolMu.Unlock()
 }
 
+// Drain the pool of any connections
+func (f *Fs) drainPool(ctx context.Context) (err error) {
+	f.poolMu.Lock()
+	defer f.poolMu.Unlock()
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Stop()
+	}
+	if len(f.pool) != 0 {
+		fs.Debugf(f, "closing %d unused connections", len(f.pool))
+	}
+	for i, c := range f.pool {
+		if cErr := c.Quit(); cErr != nil {
+			err = cErr
+		}
+		f.pool[i] = nil
+	}
+	f.pool = nil
+	return err
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
@@ -338,6 +388,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	if opt.TLS {
 		protocol = "ftps://"
 	}
+	if opt.TLS && opt.ExplicitTLS {
+		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+	}
+	var tlsConfig *tls.Config
+	if opt.TLS || opt.ExplicitTLS {
+		tlsConfig = &tls.Config{
+			ServerName:         opt.Host,
+			InsecureSkipVerify: opt.SkipVerifyTLSCert,
+		}
+	}
 	u := protocol + path.Join(dialAddr+"/", root)
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
@@ -350,10 +410,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		pass:     pass,
 		dialAddr: dialAddr,
 		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
+		tlsConf:  tlsConfig,
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
+	// set the pool drainer timer going
+	if f.opt.IdleTimeout > 0 {
+		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
+	}
 	// Make a connection and pool it to return errors early
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
@@ -382,6 +447,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	return f, err
 }
 
+// Shutdown the backend, closing any background tasks and any
+// cached connections.
+func (f *Fs) Shutdown(ctx context.Context) error {
+	return f.drainPool(ctx)
+}
+
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
 	switch errX := err.(type) {
@@ -527,7 +598,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}()
 
 	// Wait for List for up to Timeout seconds
-	timer := time.NewTimer(f.ci.Timeout)
+	timer := time.NewTimer(f.ci.TimeoutOrInfinite())
 	select {
 	case listErr = <-errchan:
 		timer.Stop()
@@ -990,5 +1061,6 @@ var (
 	_ fs.Mover       = &Fs{}
 	_ fs.DirMover    = &Fs{}
 	_ fs.PutStreamer = &Fs{}
+	_ fs.Shutdowner  = &Fs{}
 	_ fs.Object      = &Object{}
 )

backend/googlecloudstorage/googlecloudstorage.go

@@ -329,7 +329,10 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func shouldRetry(err error) (again bool, errOut error) {
+func shouldRetry(ctx context.Context, err error) (again bool, errOut error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	again = false
 	if err != nil {
 		if fserrors.ShouldRetry(err) {
@@ -455,7 +458,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			newRoot := path.Dir(f.root)
@@ -521,7 +524,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
 			objects, err = list.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			if gErr, ok := err.(*googleapi.Error); ok {
@@ -624,7 +627,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
 			buckets, err = listBuckets.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -750,7 +753,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			// Bucket already exists
@@ -785,7 +788,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
 			_, err = insertBucket.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	}, nil)
 }
@@ -802,7 +805,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
 			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	})
 }
@@ -848,7 +851,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	for {
 		err = f.pacer.Call(func() (bool, error) {
 			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -941,7 +944,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
@@ -1012,7 +1015,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = copyObject.Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1043,7 +1046,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1109,7 +1112,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = insertObject.Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1124,7 +1127,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }

backend/googlephotos/googlephotos.go

@@ -240,7 +240,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -329,7 +332,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
 	var openIDconfig map[string]interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "couldn't read openID config")
@@ -358,7 +361,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't read user info")
@@ -389,7 +392,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 	var res interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't revoke token")
@@ -476,7 +479,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't list albums")
@@ -531,7 +534,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list files")
@@ -675,7 +678,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create album")
@@ -810,7 +813,7 @@ func (o *Object) Size() int64 {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		fs.Debugf(o, "Reading size failed: %v", err)
@@ -861,7 +864,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't get media item")
@@ -938,7 +941,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -993,10 +996,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		token, err = rest.ReadBody(resp)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't upload file")
@@ -1024,7 +1027,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.BatchCreateResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create media item")
@@ -1069,7 +1072,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't delete item from album")

backend/hdfs/object.go

@@ -109,7 +109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	dirname := path.Dir(realpath)
 	fs.Debugf(o.fs, "update [%s]", realpath)
 
-	err := o.fs.client.MkdirAll(dirname, 755)
+	err := o.fs.client.MkdirAll(dirname, 0755)
 	if err != nil {
 		return err
 	}

backend/http/http_internal_test.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/rest"
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	ts := httptest.NewServer(handler)
 
 	// Configure the remote
-	config.LoadConfig(context.Background())
+	configfile.LoadConfig(context.Background())
 	// fs.Config.LogLevel = fs.LogLevelDebug
 	// fs.Config.DumpHeaders = true
 	// fs.Config.DumpBodies = true

backend/jottacloud/jottacloud.go

@@ -235,7 +235,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -615,7 +618,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -854,7 +857,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -883,7 +886,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var result api.JottaFolder
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -995,7 +998,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1101,7 +1104,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't purge directory")
@@ -1140,7 +1143,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1268,7 +1271,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var result api.JottaFile
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -1446,7 +1449,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1559,7 +1562,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var response api.AllocateFileResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1624,7 +1627,7 @@ func (o *Object) Remove(ctx context.Context) error {
 
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 

backend/local/local.go

@@ -148,6 +148,17 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_preallocate",
+			Help: `Disable preallocation of disk space for transferred files
+
+Preallocation of disk space helps prevent filesystem fragmentation.
+However, some virtual filesystem layers (such as Google Drive File
+Stream) may incorrectly set the actual file size equal to the
+preallocated space, causing checksum and file size checks to fail.
+Use this flag to disable preallocation.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name: "no_sparse",
 			Help: `Disable sparse files for multi-thread downloads
@@ -191,6 +202,7 @@ type Options struct {
 	OneFileSystem     bool                 `config:"one_file_system"`
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
+	NoPreAllocate     bool                 `config:"no_preallocate"`
 	NoSparse          bool                 `config:"no_sparse"`
 	NoSetModTime      bool                 `config:"no_set_modtime"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
@@ -1127,10 +1139,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				return err
 			}
 		}
-		// Pre-allocate the file for performance reasons
-		err = file.PreAllocate(src.Size(), f)
-		if err != nil {
-			fs.Debugf(o, "Failed to pre-allocate: %v", err)
+		if !o.fs.opt.NoPreAllocate {
+			// Pre-allocate the file for performance reasons
+			err = file.PreAllocate(src.Size(), f)
+			if err != nil {
+				fs.Debugf(o, "Failed to pre-allocate: %v", err)
+			}
 		}
 		out = f
 	} else {
@@ -1217,9 +1231,11 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 		return nil, err
 	}
 	// Pre-allocate the file for performance reasons
-	err = file.PreAllocate(size, out)
-	if err != nil {
-		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	if !f.opt.NoPreAllocate {
+		err = file.PreAllocate(size, out)
+		if err != nil {
+			fs.Debugf(o, "Failed to pre-allocate: %v", err)
+		}
 	}
 	if !f.opt.NoSparse && file.SetSparseImplemented {
 		sparseWarning.Do(func() {

backend/onedrive/onedrive.go

@@ -302,7 +302,6 @@ func init() {
 
 			m.Set(configDriveID, finalDriveID)
 			m.Set(configDriveType, rootItem.ParentReference.DriveType)
-			config.SaveConfig()
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "region",
@@ -1088,7 +1087,7 @@ func (f *Fs) Precision() time.Duration {
 
 // waitForJob waits for the job with status in url to complete
 func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
-	deadline := time.Now().Add(f.ci.Timeout)
+	deadline := time.Now().Add(f.ci.TimeoutOrInfinite())
 	for time.Now().Before(deadline) {
 		var resp *http.Response
 		var err error
@@ -1126,7 +1125,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 
 		time.Sleep(1 * time.Second)
 	}
-	return errors.Errorf("async operation didn't complete after %v", f.ci.Timeout)
+	return errors.Errorf("async operation didn't complete after %v", f.ci.TimeoutOrInfinite())
 }
 
 // Copy src to this remote using server-side copy operations.

backend/opendrive/opendrive.go

@@ -119,6 +119,7 @@ type Object struct {
 	fs      *Fs       // what this object is part of
 	remote  string    // The remote path
 	id      string    // ID of the file
+	parent  string    // ID of the parent directory
 	modTime time.Time // The modified time of the object if known
 	md5     string    // MD5 hash if known
 	size    int64     // Size of the object
@@ -206,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			Path:   "/session/login.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create session")
@@ -233,7 +234,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(ctx, remote, nil)
+		_, err := tempF.newObjectWithInfo(ctx, remote, nil, "")
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -293,7 +294,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 			Path:       "/folder/remove.json",
 		}
 		resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -388,7 +389,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -445,7 +446,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -494,7 +495,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			Path:   "/folder/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error %v", err)
@@ -517,7 +518,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, parent string) (fs.Object, error) {
 	// fs.Debugf(nil, "newObjectWithInfo(%s, %v)", remote, file)
 
 	var o *Object
@@ -526,6 +527,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (
 			fs:      f,
 			remote:  remote,
 			id:      file.FileID,
+			parent:  parent,
 			modTime: time.Unix(file.DateModified, 0),
 			size:    file.Size,
 			md5:     file.FileHash,
@@ -548,7 +550,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	// fs.Debugf(nil, "NewObject(\"%s\")", remote)
-	return f.newObjectWithInfo(ctx, remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil, "")
 }
 
 // Creates from the parameters passed in a half finished Object which
@@ -581,7 +583,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -631,7 +633,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 				Path:    "/upload/create_file.json",
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
-			return o.fs.shouldRetry(resp, err)
+			return o.fs.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create file")
@@ -657,7 +659,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -683,7 +688,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			Path:   "/folder.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -711,7 +716,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 			Path:   "/folder/list.json/" + f.session.SessionID + "/" + pathID,
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", false, errors.Wrap(err, "failed to get folder list")
@@ -754,7 +759,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	folderList := FolderList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get folder list")
@@ -768,6 +773,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		f.dirCache.Put(remote, folder.FolderID)
 		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
+		d.SetParentID(directoryID)
 		entries = append(entries, d)
 	}
 
@@ -775,7 +781,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		file.Name = f.opt.Enc.ToStandardName(file.Name)
 		// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
 		remote := path.Join(dir, file.Name)
-		o, err := f.newObjectWithInfo(ctx, remote, &file)
+		o, err := f.newObjectWithInfo(ctx, remote, &file, directoryID)
 		if err != nil {
 			return nil, err
 		}
@@ -842,7 +848,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 
 	o.modTime = modTime
@@ -862,7 +868,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open file)")
@@ -881,7 +887,7 @@ func (o *Object) Remove(ctx context.Context) error {
 			Path:       "/file.json/" + o.fs.session.SessionID + "/" + o.id,
 		}
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -910,7 +916,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:    "/upload/open_file_upload.json",
 		}
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -954,7 +960,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
-			return o.fs.shouldRetry(resp, err)
+			return o.fs.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to create file")
@@ -977,7 +983,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:   "/upload/close_file_upload.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -1003,7 +1009,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:       "/file/access.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1029,7 +1035,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 				o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get folder list")
@@ -1053,6 +1059,11 @@ func (o *Object) ID() string {
 	return o.id
 }
 
+// ParentID returns the ID of the Object parent directory if known, or "" if not
+func (o *Object) ParentID() string {
+	return o.parent
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1063,4 +1074,5 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
+	_ fs.ParentIDer      = (*Object)(nil)
 )

backend/pcloud/pcloud.go

@@ -213,7 +213,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	doRetry := false
 
 	// Check if it is an api.Error
@@ -405,7 +408,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -460,7 +463,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -597,7 +600,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -662,7 +665,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -700,7 +703,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -740,7 +743,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -787,7 +790,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -814,7 +817,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -838,7 +841,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -869,7 +872,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
 		err = q.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -927,7 +930,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1046,7 +1049,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1072,7 +1075,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1151,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// sometimes pcloud leaves a half complete file on
@@ -1181,7 +1184,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 

backend/premiumizeme/premiumizeme.go

@@ -176,7 +176,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -370,7 +373,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -407,7 +410,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -581,7 +584,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -660,7 +663,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
 		var result api.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Move http")
@@ -769,7 +772,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "CreateDir http")
@@ -896,7 +899,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -934,7 +937,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		// Just check the download URL resolves - sometimes
 		// the URLs returned by premiumize.me don't resolve so
@@ -993,7 +996,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.Response
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file http")
@@ -1035,7 +1038,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rename http")
@@ -1060,7 +1063,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "remove http")

backend/putio/error.go

@@ -1,6 +1,7 @@
 package putio
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
@@ -29,7 +30,10 @@ func (e *statusCodeError) Temporary() bool {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err == nil {
 		return false, nil
 	}

backend/putio/fs.go

@@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
 		entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return itoa(entry.ID), err
 }
@@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing file: %d", fileID)
 		children, _, err = f.client.Files.List(ctx, fileID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
@@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing files inside List: %d", parentID)
 		children, _, err = f.client.Files.List(ctx, parentID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting file: %d", fileID)
 		entry, err = f.client.Files.Get(ctx, fileID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -295,7 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 		req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
 		fs.OpenOptionAddHTTPHeaders(req.Header, options)
 		resp, err := f.oAuthClient.Do(req)
-		retry, err := shouldRetry(err)
+		retry, err := shouldRetry(ctx, err)
 		if retry {
 			return true, err
 		}
@@ -320,7 +320,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 		err = f.pacer.Call(func() (bool, error) {
 			fs.Debugf(f, "Sending zero length chunk")
 			_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		return
 	}
@@ -344,13 +344,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				// Get file offset and seek to the position
 				offset, err := f.getServerOffset(ctx, location)
 				if err != nil {
-					return shouldRetry(err)
+					return shouldRetry(ctx, err)
 				}
 				sentBytes := offset - chunkStart
 				fs.Debugf(f, "sentBytes: %d", sentBytes)
 				_, err = chunk.Seek(sentBytes, io.SeekStart)
 				if err != nil {
-					return shouldRetry(err)
+					return shouldRetry(ctx, err)
 				}
 				transferOffset = offset
 				reqSize = chunkSize - sentBytes
@@ -367,7 +367,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				offsetMismatch = true
 				return true, errors.New("connection broken")
 			}
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return
@@ -479,7 +479,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		err = f.pacer.Call(func() (bool, error) {
 			// fs.Debugf(f, "listing files: %d", dirID)
 			children, _, err = f.client.Files.List(ctx, dirID)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
@@ -493,7 +493,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "deleting file: %d", dirID)
 		err = f.client.Files.Delete(ctx, dirID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	f.dirCache.FlushDir(dir)
 	return err
@@ -552,7 +552,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -591,7 +591,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -631,7 +631,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	srcFs.dirCache.FlushDir(srcRemote)
 	return err
@@ -644,7 +644,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting account info")
 		ai, err = f.client.Account.Info(ctx)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -678,6 +678,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		}
 		// fs.Debugf(f, "emptying trash")
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 }

backend/putio/object.go

@@ -145,7 +145,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
 			return false, fs.ErrorObjectNotFound
 		}
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -220,7 +220,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var storageURL string
 	err = o.fs.pacer.Call(func() (bool, error) {
 		storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -231,7 +231,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	err = o.fs.pacer.Call(func() (bool, error) {
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil)
 		if err != nil {
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		}
 		req.Header.Set("User-Agent", o.fs.client.UserAgent)
 
@@ -241,7 +241,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		// fs.Debugf(o, "opening file: id=%d", o.file.ID)
 		resp, err = o.fs.httpClient.Do(req)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
 		_ = resp.Body.Close()
@@ -283,6 +283,6 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	return o.fs.pacer.Call(func() (bool, error) {
 		// fs.Debugf(o, "removing file: id=%d", o.file.ID)
 		err = o.fs.client.Files.Delete(ctx, o.file.ID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 }

backend/s3/s3.go

@@ -1462,7 +1462,7 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 }
 
 // s3Connection makes a connection to s3
-func s3Connection(ctx context.Context, opt *Options) (*s3.S3, *session.Session, error) {
+func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
 	// Make the auth
 	v := credentials.Value{
 		AccessKeyID:     opt.AccessKeyID,
@@ -1540,7 +1540,7 @@ func s3Connection(ctx context.Context, opt *Options) (*s3.S3, *session.Session,
 	awsConfig := aws.NewConfig().
 		WithMaxRetries(0). // Rely on rclone's retry logic
 		WithCredentials(cred).
-		WithHTTPClient(getClient(ctx, opt)).
+		WithHTTPClient(client).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
 		WithS3UseAccelerate(opt.UseAccelerateEndpoint).
 		WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
@@ -1559,9 +1559,6 @@ func s3Connection(ctx context.Context, opt *Options) (*s3.S3, *session.Session,
 	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
 		// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
 		awsSessionOpts.SharedConfigState = session.SharedConfigEnable
-		// The session constructor (aws/session/mergeConfigSrcs) will only use the user's preferred credential source
-		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
-		awsSessionOpts.Config.Credentials = nil
 	}
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
@@ -1647,7 +1644,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))
 		opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
 	}
-	c, ses, err := s3Connection(ctx, opt)
+	srv := getClient(ctx, opt)
+	c, ses, err := s3Connection(ctx, opt, srv)
 	if err != nil {
 		return nil, err
 	}
@@ -1662,7 +1660,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ses:   ses,
 		pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
 		cache: bucket.NewCache(),
-		srv:   getClient(ctx, opt),
+		srv:   srv,
 		pool: pool.New(
 			time.Duration(opt.MemoryPoolFlushTime),
 			int(opt.ChunkSize),
@@ -1697,12 +1695,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.setRoot(newRoot)
 		_, err := f.NewObject(ctx, leaf)
 		if err != nil {
-			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
-				// File doesn't exist or is a directory so return old f
-				f.setRoot(oldRoot)
-				return f, nil
-			}
-			return nil, err
+			// File doesn't exist or is a directory so return old f
+			f.setRoot(oldRoot)
+			return f, nil
 		}
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
@@ -1779,7 +1774,7 @@ func (f *Fs) updateRegionForBucket(bucket string) error {
 	// Make a new session with the new region
 	oldRegion := f.opt.Region
 	f.opt.Region = region
-	c, ses, err := s3Connection(f.ctx, &f.opt)
+	c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
 	if err != nil {
 		return errors.Wrap(err, "creating new session failed")
 	}

backend/seafile/seafile.go

@@ -372,7 +372,6 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
 			m.Set(configAuthToken, token)
 			// And delete any previous entry for password
 			m.Set(configPassword, "")
-			config.SaveConfig()
 			// And we're done here
 			break
 		}
@@ -408,7 +407,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// For 429 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.

backend/seafile/webapi.go

@@ -86,7 +86,7 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -112,7 +112,7 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -139,7 +139,7 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -170,7 +170,7 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -197,7 +197,7 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -228,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -271,7 +271,7 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -316,7 +316,7 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -358,7 +358,7 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -398,7 +398,7 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -438,7 +438,7 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -474,7 +474,7 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -505,7 +505,7 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -539,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to delete file")
@@ -565,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -614,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -659,7 +659,7 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -739,7 +739,7 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -777,7 +777,7 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -818,7 +818,7 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -860,7 +860,7 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -900,7 +900,7 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -938,7 +938,7 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -976,7 +976,7 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -1030,7 +1030,7 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -1075,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {

backend/sftp/sftp.go

@@ -204,6 +204,36 @@ Fstat instead of Stat which is called on an already open file handle.
 It has been found that this helps with IBM Sterling SFTP servers which have
 "extractability" level set to 1 which means only 1 file can be opened at
 any given time.
+`,
+			Advanced: true,
+		}, {
+			Name:    "disable_concurrent_reads",
+			Default: false,
+			Help: `If set don't use concurrent reads
+
+Normally concurrent reads are safe to use and not using them will
+degrade performance, so this option is disabled by default.
+
+Some servers limit the amount number of times a file can be
+downloaded. Using concurrent reads can trigger this limit, so if you
+have a server which returns
+
+    Failed to copy: file does not exist
+
+Then you may need to enable this flag.
+
+If concurrent reads are disabled, the use_fstat option is ignored.
+`,
+			Advanced: true,
+		}, {
+			Name:    "idle_timeout",
+			Default: fs.Duration(60 * time.Second),
+			Help: `Max time before closing idle connections
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
 `,
 			Advanced: true,
 		}},
@@ -213,27 +243,29 @@ any given time.
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string `config:"host"`
-	User              string `config:"user"`
-	Port              string `config:"port"`
-	Pass              string `config:"pass"`
-	KeyPem            string `config:"key_pem"`
-	KeyFile           string `config:"key_file"`
-	KeyFilePass       string `config:"key_file_pass"`
-	PubKeyFile        string `config:"pubkey_file"`
-	KnownHostsFile    string `config:"known_hosts_file"`
-	KeyUseAgent       bool   `config:"key_use_agent"`
-	UseInsecureCipher bool   `config:"use_insecure_cipher"`
-	DisableHashCheck  bool   `config:"disable_hashcheck"`
-	AskPassword       bool   `config:"ask_password"`
-	PathOverride      string `config:"path_override"`
-	SetModTime        bool   `config:"set_modtime"`
-	Md5sumCommand     string `config:"md5sum_command"`
-	Sha1sumCommand    string `config:"sha1sum_command"`
-	SkipLinks         bool   `config:"skip_links"`
-	Subsystem         string `config:"subsystem"`
-	ServerCommand     string `config:"server_command"`
-	UseFstat          bool   `config:"use_fstat"`
+	Host                   string      `config:"host"`
+	User                   string      `config:"user"`
+	Port                   string      `config:"port"`
+	Pass                   string      `config:"pass"`
+	KeyPem                 string      `config:"key_pem"`
+	KeyFile                string      `config:"key_file"`
+	KeyFilePass            string      `config:"key_file_pass"`
+	PubKeyFile             string      `config:"pubkey_file"`
+	KnownHostsFile         string      `config:"known_hosts_file"`
+	KeyUseAgent            bool        `config:"key_use_agent"`
+	UseInsecureCipher      bool        `config:"use_insecure_cipher"`
+	DisableHashCheck       bool        `config:"disable_hashcheck"`
+	AskPassword            bool        `config:"ask_password"`
+	PathOverride           string      `config:"path_override"`
+	SetModTime             bool        `config:"set_modtime"`
+	Md5sumCommand          string      `config:"md5sum_command"`
+	Sha1sumCommand         string      `config:"sha1sum_command"`
+	SkipLinks              bool        `config:"skip_links"`
+	Subsystem              string      `config:"subsystem"`
+	ServerCommand          string      `config:"server_command"`
+	UseFstat               bool        `config:"use_fstat"`
+	DisableConcurrentReads bool        `config:"disable_concurrent_reads"`
+	IdleTimeout            fs.Duration `config:"idle_timeout"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -251,7 +283,8 @@ type Fs struct {
 	cachedHashes *hash.Set
 	poolMu       sync.Mutex
 	pool         []*conn
-	pacer        *fs.Pacer // pacer for operations
+	drain        *time.Timer // used to drain the pool when we stop using the connections
+	pacer        *fs.Pacer   // pacer for operations
 	savedpswd    string
 }
 
@@ -360,7 +393,10 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
 		}
 	}
 	opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
-	opts = append(opts, sftp.UseFstat(f.opt.UseFstat))
+	opts = append(opts,
+		sftp.UseFstat(f.opt.UseFstat),
+		sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
+	)
 
 	return sftp.NewClientPipe(pr, pw, opts...)
 }
@@ -428,6 +464,9 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+	}
 	f.poolMu.Unlock()
 }
 
@@ -435,6 +474,12 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 func (f *Fs) drainPool(ctx context.Context) (err error) {
 	f.poolMu.Lock()
 	defer f.poolMu.Unlock()
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Stop()
+	}
+	if len(f.pool) != 0 {
+		fs.Debugf(f, "closing %d unused connections", len(f.pool))
+	}
 	for i, c := range f.pool {
 		if cErr := c.closed(); cErr == nil {
 			cErr = c.close()
@@ -667,6 +712,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 	f.mkdirLock = newStringLock()
 	f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
 	f.savedpswd = ""
+	// set the pool drainer timer going
+	if f.opt.IdleTimeout > 0 {
+		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
+	}
 
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,

backend/sharefile/sharefile.go

@@ -299,7 +299,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -324,7 +327,7 @@ func (f *Fs) readMetaDataForIDPath(ctx context.Context, id, path string, directo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &item)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
@@ -631,7 +634,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &req, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "CreateDir")
@@ -663,7 +666,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -912,7 +915,7 @@ func (f *Fs) updateItem(ctx context.Context, id, leaf, directoryID string, modTi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1133,7 +1136,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1294,7 +1297,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var dl api.DownloadSpecification
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open: fetch download specification")
@@ -1309,7 +1312,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
@@ -1365,7 +1368,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload get specification")
@@ -1390,7 +1393,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var finish api.UploadFinishResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file")
@@ -1426,7 +1429,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "remove")

backend/sharefile/upload.go

@@ -155,7 +155,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		resp, err := up.f.srv.Call(ctx, &opts)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		respBody, err = rest.ReadBody(resp)
 		// retry all errors now that the multipart upload has started

backend/sugarsync/sugarsync.go

@@ -111,7 +111,7 @@ func init() {
 			// FIXME
 			//err = f.pacer.Call(func() (bool, error) {
 			resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
-			//	return shouldRetry(resp, err)
+			//	return shouldRetry(ctx, resp, err)
 			//})
 			if err != nil {
 				log.Fatalf("Failed to get token: %v", err)
@@ -248,7 +248,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -288,7 +291,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
@@ -325,7 +328,7 @@ func (f *Fs) getAuthToken(ctx context.Context) error {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get authorization")
@@ -373,7 +376,7 @@ func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &user)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get user")
@@ -567,7 +570,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -618,7 +621,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -774,7 +777,7 @@ func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string,
 		}
 		return f.pacer.Call(func() (bool, error) {
 			resp, err := f.srv.Call(ctx, &opts)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 	}
 	// Move file/dir to deleted files if not hard delete
@@ -880,7 +883,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &copyFile, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -934,7 +937,7 @@ func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -964,7 +967,7 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err
 	var resp *http.Response
 	return f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &move, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1053,7 +1056,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var info *api.File
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1182,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1204,7 +1207,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1262,7 +1265,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to upload file")

backend/union/upstream/upstream.go

@@ -67,7 +67,7 @@ func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs
 		return nil, err
 	}
 	f := &Fs{
-		RootPath:    root,
+		RootPath:    strings.TrimRight(root, "/"),
 		writable:    true,
 		creatable:   true,
 		cacheExpiry: time.Now().Unix(),

backend/webdav/webdav.go

@@ -10,6 +10,7 @@ package webdav
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -19,20 +20,25 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/webdav/api"
 	"github.com/rclone/rclone/backend/webdav/odrvcookie"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+
+	ntlmssp "github.com/Azure/go-ntlmssp"
 )
 
 const (
@@ -42,8 +48,22 @@ const (
 	defaultDepth  = "1" // depth for PROPFIND
 )
 
+const defaultEncodingSharepointNTLM = (encoder.EncodeWin |
+	encoder.EncodeHashPercent | // required by IIS/8.5 in contrast with onedrive which doesn't need it
+	(encoder.Display &^ encoder.EncodeDot) | // test with IIS/8.5 shows that EncodeDot is not needed
+	encoder.EncodeBackSlash |
+	encoder.EncodeLeftSpace |
+	encoder.EncodeLeftTilde |
+	encoder.EncodeRightPeriod |
+	encoder.EncodeRightSpace |
+	encoder.EncodeInvalidUtf8)
+
 // Register with Fs
 func init() {
+	configEncodingHelp := fmt.Sprintf(
+		"%s\n\nDefault encoding is %s for sharepoint-ntlm or identity otherwise.",
+		config.ConfigEncodingHelp, defaultEncodingSharepointNTLM)
+
 	fs.Register(&fs.RegInfo{
 		Name:        "webdav",
 		Description: "Webdav",
@@ -67,14 +87,17 @@ func init() {
 				Help:  "Owncloud",
 			}, {
 				Value: "sharepoint",
-				Help:  "Sharepoint",
+				Help:  "Sharepoint Online, authenticated by Microsoft account.",
+			}, {
+				Value: "sharepoint-ntlm",
+				Help:  "Sharepoint with NTLM authentication. Usually self-hosted or on-premises.",
 			}, {
 				Value: "other",
 				Help:  "Other site/service or software",
 			}},
 		}, {
 			Name: "user",
-			Help: "User name",
+			Help: "User name. In case NTLM authentication is used, the username should be in the format 'Domain\\User'.",
 		}, {
 			Name:       "pass",
 			Help:       "Password.",
@@ -86,18 +109,23 @@ func init() {
 			Name:     "bearer_token_command",
 			Help:     "Command to run to get a bearer token",
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     configEncodingHelp,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	URL                string `config:"url"`
-	Vendor             string `config:"vendor"`
-	User               string `config:"user"`
-	Pass               string `config:"pass"`
-	BearerToken        string `config:"bearer_token"`
-	BearerTokenCommand string `config:"bearer_token_command"`
+	URL                string               `config:"url"`
+	Vendor             string               `config:"vendor"`
+	User               string               `config:"user"`
+	Pass               string               `config:"pass"`
+	BearerToken        string               `config:"bearer_token"`
+	BearerTokenCommand string               `config:"bearer_token_command"`
+	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote webdav
@@ -114,8 +142,10 @@ type Fs struct {
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
+	checkBeforePurge   bool          // enables extra check that directory to purge really exists
 	hasMD5             bool          // set if can use owncloud style checksums for MD5
 	hasSHA1            bool          // set if can use owncloud style checksums for SHA1
+	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
 }
 
 // Object describes a webdav object
@@ -166,7 +196,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// If we have a bearer token command and it has expired then refresh it
 	if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
 		fs.Debugf(f, "Bearer token expired: %v", err)
@@ -179,6 +212,22 @@ func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
+// safeRoundTripper is a wrapper for http.RoundTripper that serializes
+// http roundtrips. NTLM authentication sequence can involve up to four
+// rounds of negotiations and might fail due to concurrency.
+// This wrapper allows to use ntlmssp.Negotiator safely with goroutines.
+type safeRoundTripper struct {
+	fs *Fs
+	rt http.RoundTripper
+}
+
+// RoundTrip guards wrapped RoundTripper by a mutex.
+func (srt *safeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	srt.fs.ntlmAuthMu.Lock()
+	defer srt.fs.ntlmAuthMu.Unlock()
+	return srt.rt.RoundTrip(req)
+}
+
 // itemIsDir returns true if the item is a directory
 //
 // When a client sees a resourcetype it doesn't recognize it should
@@ -224,7 +273,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// does not exist
@@ -285,7 +334,11 @@ func addSlash(s string) string {
 
 // filePath returns a file path (f.root, file)
 func (f *Fs) filePath(file string) string {
-	return rest.URLPathEscape(path.Join(f.root, file))
+	subPath := path.Join(f.root, file)
+	if f.opt.Enc != encoder.EncodeZero {
+		subPath = f.opt.Enc.FromStandardPath(subPath)
+	}
+	return rest.URLPathEscape(subPath)
 }
 
 // dirPath returns a directory path (f.root, dir)
@@ -324,6 +377,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	root = strings.Trim(root, "/")
 
+	if opt.Enc == encoder.EncodeZero && opt.Vendor == "sharepoint-ntlm" {
+		opt.Enc = defaultEncodingSharepointNTLM
+	}
+
 	// Parse the endpoint
 	u, err := url.Parse(opt.URL)
 	if err != nil {
@@ -336,10 +393,28 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:         *opt,
 		endpoint:    u,
 		endpointURL: u.String(),
-		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}
+
+	client := fshttp.NewClient(ctx)
+	if opt.Vendor == "sharepoint-ntlm" {
+		// Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ ,
+		// otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED'
+		// https://docs.microsoft.com/en-us/iis/get-started/whats-new-in-iis-10/http2-on-iis says:
+		// 'Windows authentication (NTLM/Kerberos/Negotiate) is not supported with HTTP/2.'
+		t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
+			t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+		})
+
+		// Add NTLM layer
+		client.Transport = &safeRoundTripper{
+			fs: f,
+			rt: ntlmssp.Negotiator{RoundTripper: t},
+		}
+	}
+	f.srv = rest.NewClient(client).SetRoot(u.String())
+
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
@@ -465,6 +540,16 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		// to determine if we may have found a file, the request has to be resent
 		// with the depth set to 0
 		f.retryWithZeroDepth = true
+	case "sharepoint-ntlm":
+		// Sharepoint with NTLM authentication
+		// See comment above
+		f.retryWithZeroDepth = true
+
+		// Sharepoint 2016 returns status 204 to the purge request
+		// even if the directory to purge does not really exist
+		// so we must perform an extra check to detect this
+		// condition and return a proper error code.
+		f.checkBeforePurge = true
 	case "other":
 	default:
 		fs.Debugf(f, "Unknown vendor %q", vendor)
@@ -546,7 +631,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -583,7 +668,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 			fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
 			continue
 		}
-		remote := path.Join(dir, u.Path[len(baseURL.Path):])
+		subPath := u.Path[len(baseURL.Path):]
+		if f.opt.Enc != encoder.EncodeZero {
+			subPath = f.opt.Enc.ToStandardPath(subPath)
+		}
+		remote := path.Join(dir, subPath)
 		if strings.HasSuffix(remote, "/") {
 			remote = remote[:len(remote)-1]
 		}
@@ -714,7 +803,7 @@ func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return err == nil
 }
@@ -736,7 +825,7 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// Check if it already exists. The response code for this isn't
@@ -800,6 +889,21 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		if notEmpty {
 			return fs.ErrorDirectoryNotEmpty
 		}
+	} else if f.checkBeforePurge {
+		// We are doing purge as the `check` argument is unset.
+		// The quirk says that we are working with Sharepoint 2016.
+		// This provider returns status 204 even if the purged directory
+		// does not really exist so we perform an extra check here.
+		// Only the existence is checked, all other errors must be
+		// ignored here to make the rclone test suite pass.
+		depth := defaultDepth
+		if f.retryWithZeroDepth {
+			depth = "0"
+		}
+		_, err := f.readMetaDataForPath(ctx, dir, depth)
+		if err == fs.ErrorObjectNotFound {
+			return fs.ErrorDirNotFound
+		}
 	}
 	opts := rest.Opts{
 		Method:     "DELETE",
@@ -810,7 +914,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -873,7 +977,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "Copy call failed")
@@ -969,7 +1073,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "DirMove MOVE call failed")
@@ -1011,7 +1115,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about call failed")
@@ -1139,7 +1243,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1190,7 +1294,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// Give the WebDAV server a chance to get its internal state in order after the
@@ -1217,7 +1321,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 }
 

backend/webdav/webdav_test.go

@@ -38,3 +38,14 @@ func TestIntegration3(t *testing.T) {
 		NilObject:  (*webdav.Object)(nil),
 	})
 }
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration4(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavNTLM:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}

backend/yandex/yandex.go

@@ -153,7 +153,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -226,7 +229,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -468,7 +471,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// fmt.Printf("CreateDir %q Error: %s\n", path, err.Error())
@@ -537,12 +540,15 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
 		RootURL: location,
 		Method:  "GET",
 	}
-	deadline := time.Now().Add(f.ci.Timeout)
+	deadline := time.Now().Add(f.ci.TimeoutOrInfinite())
 	for time.Now().Before(deadline) {
 		var resp *http.Response
 		var body []byte
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.Call(ctx, &opts)
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			if err != nil {
 				return fserrors.ShouldRetry(err), err
 			}
@@ -568,7 +574,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
 
 		time.Sleep(1 * time.Second)
 	}
-	return errors.Errorf("async operation didn't complete after %v", f.ci.Timeout)
+	return errors.Errorf("async operation didn't complete after %v", f.ci.TimeoutOrInfinite())
 }
 
 func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err error) {
@@ -585,6 +591,9 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro
 	var body []byte
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
+		if fserrors.ContextError(ctx, &err) {
+			return false, err
+		}
 		if err != nil {
 			return fserrors.ShouldRetry(err), err
 		}
@@ -658,6 +667,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
 	var body []byte
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
+		if fserrors.ContextError(ctx, &err) {
+			return false, err
+		}
 		if err != nil {
 			return fserrors.ShouldRetry(err), err
 		}
@@ -810,7 +822,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.ErrorResponse); ok {
@@ -848,7 +860,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }
@@ -865,7 +877,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -999,7 +1011,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }
@@ -1032,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -1047,7 +1059,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1071,7 +1083,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -1089,7 +1101,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	return err

backend/zoho/zoho.go

@@ -100,7 +100,7 @@ func init() {
 				log.Fatalf("Failed to configure root directory: %v", err)
 			}
 		},
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "region",
 			Help: "Zoho region to connect to. You'll have to use the region you organization is registered in.",
 			Examples: []fs.OptionExample{{
@@ -123,7 +123,7 @@ func init() {
 				encoder.EncodeCtl |
 				encoder.EncodeDel |
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 
@@ -257,7 +257,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
@@ -354,7 +357,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -450,7 +453,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -555,7 +558,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -664,7 +667,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
 	var uploadResponse *api.UploadResponse
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "upload error")
@@ -746,7 +749,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) (err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "delete object failed")
@@ -816,7 +819,7 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e
 	var result *api.ItemInfo
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "rename failed")
@@ -869,7 +872,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *api.ItemList
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't copy file")
@@ -914,7 +917,7 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item,
 	var result *api.ItemList
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
@@ -1181,7 +1184,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err

bin/cross-compile.go

@@ -27,17 +27,22 @@ import (
 
 var (
 	// Flags
-	debug       = flag.Bool("d", false, "Print commands instead of running them.")
-	parallel    = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel.")
-	copyAs      = flag.String("release", "", "Make copies of the releases with this name")
-	gitLog      = flag.String("git-log", "", "git log to include as well")
-	include     = flag.String("include", "^.*$", "os/arch regexp to include")
-	exclude     = flag.String("exclude", "^$", "os/arch regexp to exclude")
-	cgo         = flag.Bool("cgo", false, "Use cgo for the build")
-	noClean     = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
-	tags        = flag.String("tags", "", "Space separated list of build tags")
-	buildmode   = flag.String("buildmode", "", "Passed to go build -buildmode flag")
-	compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
+	debug           = flag.Bool("d", false, "Print commands instead of running them.")
+	parallel        = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel.")
+	copyAs          = flag.String("release", "", "Make copies of the releases with this name")
+	gitLog          = flag.String("git-log", "", "git log to include as well")
+	include         = flag.String("include", "^.*$", "os/arch regexp to include")
+	exclude         = flag.String("exclude", "^$", "os/arch regexp to exclude")
+	cgo             = flag.Bool("cgo", false, "Use cgo for the build")
+	noClean         = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
+	tags            = flag.String("tags", "", "Space separated list of build tags")
+	buildmode       = flag.String("buildmode", "", "Passed to go build -buildmode flag")
+	compileOnly     = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
+	extraEnv        = flag.String("env", "", "comma separated list of VAR=VALUE env vars to set")
+	macOSSDK        = flag.String("macos-sdk", "", "macOS SDK to use")
+	macOSArch       = flag.String("macos-arch", "", "macOS arch to use")
+	extraCgoCFlags  = flag.String("cgo-cflags", "", "extra CGO_CFLAGS")
+	extraCgoLdFlags = flag.String("cgo-ldflags", "", "extra CGO_LDFLAGS")
 )
 
 // GOOS/GOARCH pairs we build for
@@ -47,6 +52,7 @@ var osarches = []string{
 	"windows/386",
 	"windows/amd64",
 	"darwin/amd64",
+	"darwin/arm64",
 	"linux/386",
 	"linux/amd64",
 	"linux/arm",
@@ -155,7 +161,15 @@ func buildZip(dir string) string {
 // Build .deb and .rpm packages
 //
 // It returns a list of artifacts it has made
-func buildDebAndRpm(dir, version, goarch string) []string {
+func buildDebAndRpm(dir, version, goarchBuild string) []string {
+	goarch := stripVersion(goarchBuild)
+
+	// Base ARM build we will mark as "arm5" so nfpm puts the
+	// architecture in as armel not armhf
+	if goarchBuild == "arm" {
+		goarch = "arm5"
+	}
+
 	// Make internal version number acceptable to .deb and .rpm
 	pkgVersion := version[1:]
 	pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
@@ -279,6 +293,15 @@ func stripVersion(goarch string) string {
 	return goarch[:i]
 }
 
+// run the command returning trimmed output
+func runOut(command ...string) string {
+	out, err := exec.Command(command[0], command[1:]...).Output()
+	if err != nil {
+		log.Fatalf("Failed to run %q: %v", command, err)
+	}
+	return strings.TrimSpace(string(out))
+}
+
 // build the binary in dir returning success or failure
 func compileArch(version, goos, goarch, dir string) bool {
 	log.Printf("Compiling %s/%s into %s", goos, goarch, dir)
@@ -314,6 +337,35 @@ func compileArch(version, goos, goarch, dir string) bool {
 		"GOOS=" + goos,
 		"GOARCH=" + stripVersion(goarch),
 	}
+	if *extraEnv != "" {
+		env = append(env, strings.Split(*extraEnv, ",")...)
+	}
+	var (
+		cgoCFlags  []string
+		cgoLdFlags []string
+	)
+	if *macOSSDK != "" {
+		flag := "-isysroot " + runOut("xcrun", "--sdk", *macOSSDK, "--show-sdk-path")
+		cgoCFlags = append(cgoCFlags, flag)
+		cgoLdFlags = append(cgoLdFlags, flag)
+	}
+	if *macOSArch != "" {
+		flag := "-arch " + *macOSArch
+		cgoCFlags = append(cgoCFlags, flag)
+		cgoLdFlags = append(cgoLdFlags, flag)
+	}
+	if *extraCgoCFlags != "" {
+		cgoCFlags = append(cgoCFlags, *extraCgoCFlags)
+	}
+	if *extraCgoLdFlags != "" {
+		cgoLdFlags = append(cgoLdFlags, *extraCgoLdFlags)
+	}
+	if len(cgoCFlags) > 0 {
+		env = append(env, "CGO_CFLAGS="+strings.Join(cgoCFlags, " "))
+	}
+	if len(cgoLdFlags) > 0 {
+		env = append(env, "CGO_LDFLAGS="+strings.Join(cgoLdFlags, " "))
+	}
 	if !*cgo {
 		env = append(env, "CGO_ENABLED=0")
 	} else {
@@ -332,7 +384,7 @@ func compileArch(version, goos, goarch, dir string) bool {
 			artifacts := []string{buildZip(dir)}
 			// build a .deb and .rpm if appropriate
 			if goos == "linux" {
-				artifacts = append(artifacts, buildDebAndRpm(dir, version, stripVersion(goarch))...)
+				artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
 			}
 			if *copyAs != "" {
 				for _, artifact := range artifacts {

bin/make_test_files.go

@@ -1,146 +0,0 @@
-// +build ignore
-
-// Build a directory structure with the required number of files in
-//
-// Run with go run make_test_files.go [flag] <directory>
-package main
-
-import (
-	cryptrand "crypto/rand"
-	"flag"
-	"io"
-	"log"
-	"math/rand"
-	"os"
-	"path/filepath"
-)
-
-var (
-	// Flags
-	numberOfFiles            = flag.Int("n", 1000, "Number of files to create")
-	averageFilesPerDirectory = flag.Int("files-per-directory", 10, "Average number of files per directory")
-	maxDepth                 = flag.Int("max-depth", 10, "Maximum depth of directory hierarchy")
-	minFileSize              = flag.Int64("min-size", 0, "Minimum size of file to create")
-	maxFileSize              = flag.Int64("max-size", 100, "Maximum size of files to create")
-	minFileNameLength        = flag.Int("min-name-length", 4, "Minimum size of file to create")
-	maxFileNameLength        = flag.Int("max-name-length", 12, "Maximum size of files to create")
-
-	directoriesToCreate int
-	totalDirectories    int
-	fileNames           = map[string]struct{}{} // keep a note of which file name we've used already
-)
-
-// randomString create a random string for test purposes
-func randomString(n int) string {
-	const (
-		vowel     = "aeiou"
-		consonant = "bcdfghjklmnpqrstvwxyz"
-		digit     = "0123456789"
-	)
-	pattern := []string{consonant, vowel, consonant, vowel, consonant, vowel, consonant, digit}
-	out := make([]byte, n)
-	p := 0
-	for i := range out {
-		source := pattern[p]
-		p = (p + 1) % len(pattern)
-		out[i] = source[rand.Intn(len(source))]
-	}
-	return string(out)
-}
-
-// fileName creates a unique random file or directory name
-func fileName() (name string) {
-	for {
-		length := rand.Intn(*maxFileNameLength-*minFileNameLength) + *minFileNameLength
-		name = randomString(length)
-		if _, found := fileNames[name]; !found {
-			break
-		}
-	}
-	fileNames[name] = struct{}{}
-	return name
-}
-
-// dir is a directory in the directory hierarchy being built up
-type dir struct {
-	name     string
-	depth    int
-	children []*dir
-	parent   *dir
-}
-
-// Create a random directory hierarchy under d
-func (d *dir) createDirectories() {
-	for totalDirectories < directoriesToCreate {
-		newDir := &dir{
-			name:   fileName(),
-			depth:  d.depth + 1,
-			parent: d,
-		}
-		d.children = append(d.children, newDir)
-		totalDirectories++
-		switch rand.Intn(4) {
-		case 0:
-			if d.depth < *maxDepth {
-				newDir.createDirectories()
-			}
-		case 1:
-			return
-		}
-	}
-	return
-}
-
-// list the directory hierarchy
-func (d *dir) list(path string, output []string) []string {
-	dirPath := filepath.Join(path, d.name)
-	output = append(output, dirPath)
-	for _, subDir := range d.children {
-		output = subDir.list(dirPath, output)
-	}
-	return output
-}
-
-// writeFile writes a random file at dir/name
-func writeFile(dir, name string) {
-	err := os.MkdirAll(dir, 0777)
-	if err != nil {
-		log.Fatalf("Failed to make directory %q: %v", dir, err)
-	}
-	path := filepath.Join(dir, name)
-	fd, err := os.Create(path)
-	if err != nil {
-		log.Fatalf("Failed to open file %q: %v", path, err)
-	}
-	size := rand.Int63n(*maxFileSize-*minFileSize) + *minFileSize
-	_, err = io.CopyN(fd, cryptrand.Reader, size)
-	if err != nil {
-		log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
-	}
-	err = fd.Close()
-	if err != nil {
-		log.Fatalf("Failed to close file %q: %v", path, err)
-	}
-}
-
-func main() {
-	flag.Parse()
-	args := flag.Args()
-	if len(args) != 1 {
-		log.Fatalf("Require 1 directory argument")
-	}
-	outputDirectory := args[0]
-	log.Printf("Output dir %q", outputDirectory)
-
-	directoriesToCreate = *numberOfFiles / *averageFilesPerDirectory
-	log.Printf("directoriesToCreate %v", directoriesToCreate)
-	root := &dir{name: outputDirectory, depth: 1}
-	for totalDirectories < directoriesToCreate {
-		root.createDirectories()
-	}
-	dirs := root.list("", []string{})
-	for i := 0; i < *numberOfFiles; i++ {
-		dir := dirs[rand.Intn(len(dirs))]
-		writeFile(dir, fileName())
-	}
-}

cmd/all/all.go

@@ -25,7 +25,6 @@ import (
 	_ "github.com/rclone/rclone/cmd/genautocomplete"
 	_ "github.com/rclone/rclone/cmd/gendocs"
 	_ "github.com/rclone/rclone/cmd/hashsum"
-	_ "github.com/rclone/rclone/cmd/info"
 	_ "github.com/rclone/rclone/cmd/link"
 	_ "github.com/rclone/rclone/cmd/listremotes"
 	_ "github.com/rclone/rclone/cmd/ls"
@@ -34,7 +33,6 @@ import (
 	_ "github.com/rclone/rclone/cmd/lsjson"
 	_ "github.com/rclone/rclone/cmd/lsl"
 	_ "github.com/rclone/rclone/cmd/md5sum"
-	_ "github.com/rclone/rclone/cmd/memtest"
 	_ "github.com/rclone/rclone/cmd/mkdir"
 	_ "github.com/rclone/rclone/cmd/mount"
 	_ "github.com/rclone/rclone/cmd/mount2"
@@ -49,11 +47,17 @@ import (
 	_ "github.com/rclone/rclone/cmd/reveal"
 	_ "github.com/rclone/rclone/cmd/rmdir"
 	_ "github.com/rclone/rclone/cmd/rmdirs"
+	_ "github.com/rclone/rclone/cmd/selfupdate"
 	_ "github.com/rclone/rclone/cmd/serve"
 	_ "github.com/rclone/rclone/cmd/settier"
 	_ "github.com/rclone/rclone/cmd/sha1sum"
 	_ "github.com/rclone/rclone/cmd/size"
 	_ "github.com/rclone/rclone/cmd/sync"
+	_ "github.com/rclone/rclone/cmd/test"
+	_ "github.com/rclone/rclone/cmd/test/histogram"
+	_ "github.com/rclone/rclone/cmd/test/info"
+	_ "github.com/rclone/rclone/cmd/test/makefiles"
+	_ "github.com/rclone/rclone/cmd/test/memory"
 	_ "github.com/rclone/rclone/cmd/touch"
 	_ "github.com/rclone/rclone/cmd/tree"
 	_ "github.com/rclone/rclone/cmd/version"

cmd/authorize/authorize.go

@@ -29,8 +29,8 @@ rclone config.
 
 Use the --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.`,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 3, command, args)
-		config.Authorize(context.Background(), args, noAutoBrowser)
+		return config.Authorize(context.Background(), args, noAutoBrowser)
 	},
 }

cmd/cmd.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/config/configflags"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/filter"
@@ -382,6 +383,12 @@ func initConfig() {
 	// Finish parsing any command line flags
 	configflags.SetFlags(ci)
 
+	// Load the config
+	configfile.LoadConfig(ctx)
+
+	// Start accounting
+	accounting.Start(ctx)
+
 	// Hide console window
 	if ci.NoConsole {
 		terminal.HideConsole()
@@ -541,6 +548,9 @@ func Main() {
 	setupRootCommand(Root)
 	AddBackendFlags()
 	if err := Root.Execute(); err != nil {
+		if strings.HasPrefix(err.Error(), "unknown command") {
+			Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath())
+		}
 		log.Fatalf("Fatal error: %v", err)
 	}
 }

cmd/cmount/mount.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -36,6 +37,19 @@ func init() {
 	mountlib.AddRc("cmount", mount)
 }
 
+// Find the option string in the current options
+func findOption(name string, options []string) (found bool) {
+	for _, option := range options {
+		if option == "-o" {
+			continue
+		}
+		if strings.Contains(option, name) {
+			return true
+		}
+	}
+	return false
+}
+
 // mountOptions configures the options from the command line flags
 func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
 	// Options
@@ -105,6 +119,13 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.
 	for _, option := range opt.ExtraFlags {
 		options = append(options, option)
 	}
+	if runtime.GOOS == "darwin" {
+		if !findOption("modules=iconv", options) {
+			iconv := "modules=iconv,from_code=UTF-8,to_code=UTF-8-MAC"
+			options = append(options, "-o", iconv)
+			fs.Debugf(nil, "Adding \"-o %s\" for macOS", iconv)
+		}
+	}
 	return options
 }
 

cmd/cmount/mountpoint_windows.go

@@ -103,8 +103,9 @@ func handleLocalMountpath(mountpath string, opt *mountlib.Options) (string, erro
 	} else if !os.IsNotExist(err) {
 		return "", errors.Wrap(err, "failed to retrieve mountpoint path information")
 	}
-	//if isDriveRootPath(mountpath) { // Assume intention with "X:\" was "X:"
-	//	mountpoint = mountpath[:len(mountpath)-1] // WinFsp needs drive mountpoints without trailing path separator
+	if isDriveRootPath(mountpath) { // Assume intention with "X:\" was "X:"
+		mountpath = mountpath[:len(mountpath)-1] // WinFsp needs drive mountpoints without trailing path separator
+	}
 	if !isDrive(mountpath) {
 		// Assuming directory path, since it is not a pure drive letter string such as "X:".
 		// Drive letter string can be used as is, since we have already checked it does not exist,
@@ -113,14 +114,12 @@ func handleLocalMountpath(mountpath string, opt *mountlib.Options) (string, erro
 			fs.Errorf(nil, "Ignoring --network-mode as it is not supported with directory mountpoint")
 			opt.NetworkMode = false
 		}
+		var err error
+		if mountpath, err = filepath.Abs(mountpath); err != nil { // Ensures parent is found but also more informative log messages
+			return "", errors.Wrap(err, "mountpoint path is not valid: "+mountpath)
+		}
 		parent := filepath.Join(mountpath, "..")
-		if parent == "" || parent == "." {
-			return "", errors.New("mountpoint directory is not valid: " + parent)
-		}
-		if os.IsPathSeparator(parent[len(parent)-1]) { // Ends in a separator only if it is the root directory
-			return "", errors.New("mountpoint directory is at root: " + parent)
-		}
-		if _, err := os.Stat(parent); err != nil {
+		if _, err = os.Stat(parent); err != nil {
 			if os.IsNotExist(err) {
 				return "", errors.New("parent of mountpoint directory does not exist: " + parent)
 			}

cmd/dedupe/dedupe.go

@@ -32,8 +32,8 @@ By default ` + "`dedupe`" + ` interactively finds files with duplicate
 names and offers to delete all but one or rename them to be
 different. This is known as deduping by name.
 
-Deduping by name is only useful with backends like Google Drive which
-can have duplicate file names. It can be run on wrapping backends
+Deduping by name is only useful with a small group of backends (e.g. Google Drive,
+Opendrive) that can have duplicate file names. It can be run on wrapping backends
 (e.g. crypt) if they wrap a backend which supports duplicate file
 names.
 

cmd/listremotes/listremotes.go

@@ -41,7 +41,7 @@ When uses with the -l flag it lists the types too.
 		}
 		for _, remote := range remotes {
 			if listLong {
-				remoteType := config.FileGet(remote, "type", "UNKNOWN")
+				remoteType := config.FileGet(remote, "type")
 				fmt.Printf("%-*s %s\n", maxlen+1, remote+":", remoteType)
 			} else {
 				fmt.Printf("%s:\n", remote)

cmd/mountlib/mount.go

@@ -71,7 +71,7 @@ const (
 func init() {
 	// DaemonTimeout defaults to non zero for macOS
 	if runtime.GOOS == "darwin" {
-		DefaultOpt.DaemonTimeout = 15 * time.Minute
+		DefaultOpt.DaemonTimeout = 10 * time.Minute
 	}
 }
 
@@ -179,15 +179,15 @@ is an **empty** **existing** directory:
 
 On Windows you can start a mount in different ways. See [below](#mounting-modes-on-windows)
 for details. The following examples will mount to an automatically assigned drive,
-to specific drive letter |X:|, to path |C:\path\to\nonexistent\directory|
-(which must be **non-existent** subdirectory of an **existing** parent directory or drive,
+to specific drive letter |X:|, to path |C:\path\parent\mount|
+(where parent directory or drive must exist, and mount must **not** exist,
 and is not supported when [mounting as a network drive](#mounting-modes-on-windows)), and
 the last example will mount as network share |\\cloud\remote| and map it to an
 automatically assigned drive:
 
     rclone @ remote:path/to/files *
     rclone @ remote:path/to/files X:
-    rclone @ remote:path/to/files C:\path\to\nonexistent\directory
+    rclone @ remote:path/to/files C:\path\parent\mount
     rclone @ remote:path/to/files \\cloud\remote
 
 When the program ends while in foreground mode, either via Ctrl+C or receiving
@@ -241,14 +241,14 @@ and experience unexpected program errors, freezes or other issues, consider moun
 as a network drive instead.
 
 When mounting as a fixed disk drive you can either mount to an unused drive letter,
-or to a path - which must be **non-existent** subdirectory of an **existing** parent
+or to a path representing a **non-existent** subdirectory of an **existing** parent
 directory or drive. Using the special value |*| will tell rclone to
 automatically assign the next available drive letter, starting with Z: and moving backward.
 Examples:
 
     rclone @ remote:path/to/files *
     rclone @ remote:path/to/files X:
-    rclone @ remote:path/to/files C:\path\to\nonexistent\directory
+    rclone @ remote:path/to/files C:\path\parent\mount
     rclone @ remote:path/to/files X:
 
 Option |--volname| can be used to set a custom volume name for the mounted
@@ -321,10 +321,24 @@ Note that the mapping of permissions is not always trivial, and the result
 you see in Windows Explorer may not be exactly like you expected.
 For example, when setting a value that includes write access, this will be
 mapped to individual permissions "write attributes", "write data" and "append data",
-but not "write extended attributes" (WinFsp does not support extended attributes,
-see [this](https://github.com/billziss-gh/winfsp/wiki/NTFS-Compatibility)).
-Windows will then show this as basic permission "Special" instead of "Write",
-because "Write" includes the "write extended attributes" permission.
+but not "write extended attributes". Windows will then show this as basic
+permission "Special" instead of "Write", because "Write" includes the
+"write extended attributes" permission.
+
+If you set POSIX permissions for only allowing access to the owner, using
+|--file-perms 0600 --dir-perms 0700|, the user group and the built-in "Everyone"
+group will still be given some special permissions, such as "read attributes"
+and "read permissions", in Windows. This is done for compatibility reasons,
+e.g. to allow users without additional permissions to be able to read basic
+metadata about files like in UNIX. One case that may arise is that other programs
+(incorrectly) interprets this as the file being accessible by everyone. For example
+an SSH client may warn about "unprotected private key file".
+
+WinFsp 2021 (version 1.9, still in beta) introduces a new FUSE option "FileSecurity",
+that allows the complete specification of file security descriptors using
+[SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
+With this you can work around issues such as the mentioned "unprotected private key file"
+by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to the owner (OW).
 
 #### Windows caveats
 
@@ -348,7 +362,7 @@ Without the use of |--vfs-cache-mode| this can only write files
 sequentially, it can only seek when reading.  This means that many
 applications won't work with their files on an rclone mount without
 |--vfs-cache-mode writes| or |--vfs-cache-mode full|.
-See the [File Caching](#file-caching) section for more info.
+See the [VFS File Caching](#vfs-file-caching) section for more info.
 
 The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2,
 Hubic) do not support the concept of empty directories, so empty
@@ -363,7 +377,7 @@ File systems expect things to be 100% reliable, whereas cloud storage
 systems are a long way from 100% reliable. The rclone sync/copy
 commands cope with this with lots of retries.  However rclone @
 can't use retries in the same way without making local copies of the
-uploads. Look at the [file caching](#file-caching)
+uploads. Look at the [VFS File Caching](#vfs-file-caching)
 for solutions to make @ more reliable.
 
 ### Attribute caching

cmd/mountlib/rc_test.go

@@ -13,6 +13,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/cmount"
 	_ "github.com/rclone/rclone/cmd/mount"
 	_ "github.com/rclone/rclone/cmd/mount2"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -20,6 +21,7 @@ import (
 
 func TestRc(t *testing.T) {
 	ctx := context.Background()
+	configfile.LoadConfig(ctx)
 	mount := rc.Calls.Get("mount/mount")
 	assert.NotNil(t, mount)
 	unmount := rc.Calls.Get("mount/unmount")

cmd/rc/rc.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/rc/jobs"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -164,7 +165,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err
 		if call == nil {
 			return nil, errors.Errorf("method %q not found", path)
 		}
-		out, err = call.Fn(context.Background(), in)
+		_, out, err := jobs.NewJob(ctx, call.Fn, in)
 		if err != nil {
 			return nil, errors.Wrap(err, "loopback call failed")
 		}

cmd/rcd/rcd.go

@@ -3,10 +3,13 @@ package rcd
 import (
 	"context"
 	"log"
+	"sync"
 
+	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/spf13/cobra"
 )
 
@@ -48,6 +51,22 @@ See the [rc documentation](/rc/) for more info on the rc flags.
 			log.Fatal("rc server not configured")
 		}
 
+		// Notify stopping on exit
+		var finaliseOnce sync.Once
+		finalise := func() {
+			finaliseOnce.Do(func() {
+				_ = sysdnotify.Stopping()
+			})
+		}
+		fnHandle := atexit.Register(finalise)
+		defer atexit.Unregister(fnHandle)
+
+		// Notify ready to systemd
+		if err := sysdnotify.Ready(); err != nil {
+			log.Fatalf("failed to notify ready to systemd: %v", err)
+		}
+
 		s.Wait()
+		finalise()
 	},
 }

cmd/selfupdate/help.go

@@ -0,0 +1,22 @@
+package selfupdate
+
+// Note: "|" will be replaced by backticks in the help string below
+var selfUpdateHelp string = `
+This command downloads the latest release of rclone and replaces
+the currently running binary. The download is verified with a hashsum
+and cryptographically signed signature.
+
+The |--version VER| flag, if given, will update to a concrete version
+instead of the latest one. If you omit micro version from |VER| (for
+example |1.53|), the latest matching micro version will be used.
+
+If you previously installed rclone via a package manager, the package may
+include local documentation or configure services. You may wish to update
+with the flag |--package deb| or |--package rpm| (whichever is correct for
+your OS) to update these too. This command with the default |--package zip|
+will update only the rclone executable so the local manual may become
+inaccurate after it.
+
+Note: Windows forbids deletion of a currently running executable so this
+command will rename the old executable to 'rclone.old.exe' upon success.
+`

cmd/selfupdate/selfupdate.go

@@ -0,0 +1,474 @@
+package selfupdate
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/spf13/cobra"
+
+	versionCmd "github.com/rclone/rclone/cmd/version"
+)
+
+// Options contains options for the self-update command
+type Options struct {
+	Check   bool
+	Output  string // output path
+	Beta    bool   // mutually exclusive with Stable (false means "stable")
+	Stable  bool   // mutually exclusive with Beta
+	Version string
+	Package string // package format: zip, deb, rpm (empty string means "zip")
+}
+
+// Opt is options set via command line
+var Opt = Options{}
+
+func init() {
+	cmd.Root.AddCommand(cmdSelfUpdate)
+	cmdFlags := cmdSelfUpdate.Flags()
+	flags.BoolVarP(cmdFlags, &Opt.Check, "check", "", Opt.Check, "Check for latest release, do not download.")
+	flags.StringVarP(cmdFlags, &Opt.Output, "output", "", Opt.Output, "Save the downloaded binary at a given path (default: replace running binary)")
+	flags.BoolVarP(cmdFlags, &Opt.Stable, "stable", "", Opt.Stable, "Install stable release (this is the default)")
+	flags.BoolVarP(cmdFlags, &Opt.Beta, "beta", "", Opt.Beta, "Install beta release.")
+	flags.StringVarP(cmdFlags, &Opt.Version, "version", "", Opt.Version, "Install the given rclone version (default: latest)")
+	flags.StringVarP(cmdFlags, &Opt.Package, "package", "", Opt.Package, "Package format: zip|deb|rpm (default: zip)")
+}
+
+var cmdSelfUpdate = &cobra.Command{
+	Use:     "selfupdate",
+	Aliases: []string{"self-update"},
+	Short:   `Update the rclone binary.`,
+	Long:    strings.ReplaceAll(selfUpdateHelp, "|", "`"),
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 0, command, args)
+		if Opt.Package == "" {
+			Opt.Package = "zip"
+		}
+		gotActionFlags := Opt.Stable || Opt.Beta || Opt.Output != "" || Opt.Version != "" || Opt.Package != "zip"
+		if Opt.Check && !gotActionFlags {
+			versionCmd.CheckVersion()
+			return
+		}
+		if Opt.Package != "zip" {
+			if Opt.Package != "deb" && Opt.Package != "rpm" {
+				log.Fatalf("--package should be one of zip|deb|rpm")
+			}
+			if runtime.GOOS != "linux" {
+				log.Fatalf(".deb and .rpm packages are supported only on Linux")
+			} else if os.Geteuid() != 0 && !Opt.Check {
+				log.Fatalf(".deb and .rpm must be installed by root")
+			}
+			if Opt.Output != "" && !Opt.Check {
+				fmt.Println("Warning: --output is ignored with --package deb|rpm")
+			}
+		}
+		if err := InstallUpdate(context.Background(), &Opt); err != nil {
+			log.Fatalf("Error: %v", err)
+		}
+	},
+}
+
+// GetVersion can get the latest release number from the download site
+// or massage a stable release number - prepend semantic "v" prefix
+// or find the latest micro release for a given major.minor release.
+// Note: this will not be applied to beta releases.
+func GetVersion(ctx context.Context, beta bool, version string) (newVersion, siteURL string, err error) {
+	siteURL = "https://downloads.rclone.org"
+	if beta {
+		siteURL = "https://beta.rclone.org"
+	}
+
+	if version == "" {
+		// Request the latest release number from the download site
+		_, newVersion, _, err = versionCmd.GetVersion(siteURL + "/version.txt")
+		return
+	}
+
+	newVersion = version
+	if version[0] != 'v' {
+		newVersion = "v" + version
+	}
+	if beta {
+		return
+	}
+
+	if valid, _ := regexp.MatchString(`^v\d+\.\d+(\.\d+)?$`, newVersion); !valid {
+		return "", siteURL, errors.New("invalid semantic version")
+	}
+
+	// Find the latest stable micro release
+	if strings.Count(newVersion, ".") == 1 {
+		html, err := downloadFile(ctx, siteURL)
+		if err != nil {
+			return "", siteURL, errors.Wrap(err, "failed to get list of releases")
+		}
+		reSubver := fmt.Sprintf(`href="\./%s\.\d+/"`, regexp.QuoteMeta(newVersion))
+		allSubvers := regexp.MustCompile(reSubver).FindAllString(string(html), -1)
+		if allSubvers == nil {
+			return "", siteURL, errors.New("could not find the minor release")
+		}
+		// Use the fact that releases in the index are sorted by date
+		lastSubver := allSubvers[len(allSubvers)-1]
+		newVersion = lastSubver[8 : len(lastSubver)-2]
+	}
+	return
+}
+
+// InstallUpdate performs rclone self-update
+func InstallUpdate(ctx context.Context, opt *Options) error {
+	// Find the latest release number
+	if opt.Stable && opt.Beta {
+		return errors.New("--stable and --beta are mutually exclusive")
+	}
+
+	newVersion, siteURL, err := GetVersion(ctx, opt.Beta, opt.Version)
+	if err != nil {
+		return errors.Wrap(err, "unable to detect new version")
+	}
+
+	if newVersion == "" {
+		var err error
+		_, newVersion, _, err = versionCmd.GetVersion(siteURL + "/version.txt")
+		if err != nil {
+			return errors.Wrap(err, "unable to detect new version")
+		}
+	}
+
+	if newVersion == fs.Version {
+		fmt.Println("rclone is up to date")
+		return nil
+	}
+
+	// Install .deb/.rpm package if requested by user
+	if opt.Package == "deb" || opt.Package == "rpm" {
+		if opt.Check {
+			fmt.Println("Warning: --package flag is ignored in --check mode")
+		} else {
+			err := installPackage(ctx, opt.Beta, newVersion, siteURL, opt.Package)
+			if err == nil {
+				fmt.Printf("Successfully updated rclone package to version %s\n", newVersion)
+			}
+			return err
+		}
+	}
+
+	// Get the current executable path
+	executable, err := os.Executable()
+	if err != nil {
+		return errors.Wrap(err, "unable to find executable")
+	}
+
+	targetFile := opt.Output
+	if targetFile == "" {
+		targetFile = executable
+	}
+
+	if opt.Check {
+		fmt.Printf("Without --check this would install rclone version %s at %s\n", newVersion, targetFile)
+		return nil
+	}
+
+	// Make temporary file names and check for possible access errors in advance
+	var newFile string
+	if newFile, err = makeRandomExeName(targetFile, "new"); err != nil {
+		return err
+	}
+	savedFile := ""
+	if runtime.GOOS == "windows" {
+		savedFile = targetFile
+		if strings.HasSuffix(savedFile, ".exe") {
+			savedFile = savedFile[:len(savedFile)-4]
+		}
+		savedFile += ".old.exe"
+	}
+
+	if savedFile == executable || newFile == executable {
+		return fmt.Errorf("%s: a temporary file would overwrite the executable, specify a different --output path", targetFile)
+	}
+
+	if err := verifyAccess(targetFile); err != nil {
+		return err
+	}
+
+	// Download the update as a temporary file
+	err = downloadUpdate(ctx, opt.Beta, newVersion, siteURL, newFile, "zip")
+	if err != nil {
+		return errors.Wrap(err, "failed to update rclone")
+	}
+
+	err = replaceExecutable(targetFile, newFile, savedFile)
+	if err == nil {
+		fmt.Printf("Successfully updated rclone to version %s\n", newVersion)
+	}
+	return err
+}
+
+func installPackage(ctx context.Context, beta bool, version, siteURL, packageFormat string) error {
+	tempFile, err := ioutil.TempFile("", "rclone.*."+packageFormat)
+	if err != nil {
+		return errors.Wrap(err, "unable to write temporary package")
+	}
+	packageFile := tempFile.Name()
+	_ = tempFile.Close()
+	defer func() {
+		if rmErr := os.Remove(packageFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary package: %v", packageFile, rmErr)
+		}
+	}()
+	if err := downloadUpdate(ctx, beta, version, siteURL, packageFile, packageFormat); err != nil {
+		return err
+	}
+
+	packageCommand := "dpkg"
+	if packageFormat == "rpm" {
+		packageCommand = "rpm"
+	}
+	cmd := exec.Command(packageCommand, "-i", packageFile)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run %s: %v", packageCommand, err)
+	}
+	return nil
+}
+
+func replaceExecutable(targetFile, newFile, savedFile string) error {
+	// Copy permission bits from the old executable
+	// (it was extracted with mode 0755)
+	fileInfo, err := os.Lstat(targetFile)
+	if err == nil {
+		if err = os.Chmod(newFile, fileInfo.Mode()); err != nil {
+			return errors.Wrap(err, "failed to set permission")
+		}
+	}
+
+	if err = os.Remove(targetFile); os.IsNotExist(err) {
+		err = nil
+	}
+
+	if err != nil && savedFile != "" {
+		// Windows forbids removal of a running executable so we rename it.
+		// For starters, rename download as the original file with ".old.exe" appended.
+		var saveErr error
+		if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
+			saveErr = nil
+		}
+		if saveErr == nil {
+			saveErr = os.Rename(targetFile, savedFile)
+		}
+		if saveErr != nil {
+			// The ".old" file cannot be removed or cannot be renamed to.
+			// This usually means that the running executable has a name with ".old".
+			// This can happen in very rare cases, but we ought to handle it.
+			// Try inserting a randomness in the name to mitigate it.
+			fs.Debugf(nil, "%s: cannot replace old file, randomizing name", savedFile)
+
+			savedFile, saveErr = makeRandomExeName(targetFile, "old")
+			if saveErr == nil {
+				if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
+					saveErr = nil
+				}
+			}
+			if saveErr == nil {
+				saveErr = os.Rename(targetFile, savedFile)
+			}
+		}
+		if saveErr == nil {
+			fmt.Printf("The old executable was saved as %s\n", savedFile)
+			err = nil
+		}
+	}
+
+	if err == nil {
+		err = os.Rename(newFile, targetFile)
+	}
+	if err != nil {
+		if rmErr := os.Remove(newFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
+		}
+		return err
+	}
+	return nil
+}
+
+func makeRandomExeName(baseName, extension string) (string, error) {
+	const maxAttempts = 5
+
+	if runtime.GOOS == "windows" {
+		if strings.HasSuffix(baseName, ".exe") {
+			baseName = baseName[:len(baseName)-4]
+		}
+		extension += ".exe"
+	}
+
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		filename := fmt.Sprintf("%s.%s.%s", baseName, random.String(4), extension)
+		if _, err := os.Stat(filename); os.IsNotExist(err) {
+			return filename, nil
+		}
+	}
+
+	return "", fmt.Errorf("cannot find a file name like %s.xxxx.%s", baseName, extension)
+}
+
+func downloadUpdate(ctx context.Context, beta bool, version, siteURL, newFile, packageFormat string) error {
+	osName := runtime.GOOS
+	arch := runtime.GOARCH
+	if arch == "darwin" {
+		arch = "osx"
+	}
+
+	archiveFilename := fmt.Sprintf("rclone-%s-%s-%s.%s", version, osName, arch, packageFormat)
+	archiveURL := fmt.Sprintf("%s/%s/%s", siteURL, version, archiveFilename)
+	archiveBuf, err := downloadFile(ctx, archiveURL)
+	if err != nil {
+		return err
+	}
+	gotHash := sha256.Sum256(archiveBuf)
+	strHash := hex.EncodeToString(gotHash[:])
+	fs.Debugf(nil, "downloaded release archive with hashsum %s from %s", strHash, archiveURL)
+
+	// CI/CD does not provide hashsums for beta releases
+	if !beta {
+		if err := verifyHashsum(ctx, siteURL, version, archiveFilename, gotHash[:]); err != nil {
+			return err
+		}
+	}
+
+	if packageFormat == "deb" || packageFormat == "rpm" {
+		if err := ioutil.WriteFile(newFile, archiveBuf, 0644); err != nil {
+			return errors.Wrap(err, "cannot write temporary ."+packageFormat)
+		}
+		return nil
+	}
+
+	entryName := fmt.Sprintf("rclone-%s-%s-%s/rclone", version, osName, arch)
+	if runtime.GOOS == "windows" {
+		entryName += ".exe"
+	}
+
+	// Extract executable to a temporary file, then replace it by an instant rename
+	err = extractZipToFile(archiveBuf, entryName, newFile)
+	if err != nil {
+		return err
+	}
+	fs.Debugf(nil, "extracted %s to %s", entryName, newFile)
+	return nil
+}
+
+func verifyAccess(file string) error {
+	admin := "root"
+	if runtime.GOOS == "windows" {
+		admin = "Administrator"
+	}
+
+	fileInfo, fileErr := os.Lstat(file)
+
+	if fileErr != nil {
+		dir := filepath.Dir(file)
+		dirInfo, dirErr := os.Lstat(dir)
+		if dirErr != nil {
+			return dirErr
+		}
+		if !dirInfo.Mode().IsDir() {
+			return fmt.Errorf("%s: parent path is not a directory, specify a different path using --output", dir)
+		}
+		if !writable(dir) {
+			return fmt.Errorf("%s: directory is not writable, please run self-update as %s", dir, admin)
+		}
+	}
+
+	if fileErr == nil && !fileInfo.Mode().IsRegular() {
+		return fmt.Errorf("%s: path is not a normal file, specify a different path using --output", file)
+	}
+
+	if fileErr == nil && !writable(file) {
+		return fmt.Errorf("%s: file is not writable, run self-update as %s", file, admin)
+	}
+
+	return nil
+}
+
+func findFileHash(buf []byte, filename string) (hash []byte, err error) {
+	lines := bufio.NewScanner(bytes.NewReader(buf))
+	for lines.Scan() {
+		tokens := strings.Split(lines.Text(), "  ")
+		if len(tokens) == 2 && tokens[1] == filename {
+			if hash, err := hex.DecodeString(tokens[0]); err == nil {
+				return hash, nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("%s: unable to find hash", filename)
+}
+
+func extractZipToFile(buf []byte, entryName, newFile string) error {
+	zipReader, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
+	if err != nil {
+		return err
+	}
+
+	var reader io.ReadCloser
+	for _, entry := range zipReader.File {
+		if entry.Name == entryName {
+			reader, err = entry.Open()
+			break
+		}
+	}
+	if reader == nil || err != nil {
+		return fmt.Errorf("%s: file not found in archive", entryName)
+	}
+	defer func() {
+		_ = reader.Close()
+	}()
+
+	err = os.Remove(newFile)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("%s: unable to create new file: %v", newFile, err)
+	}
+	writer, err := os.OpenFile(newFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(writer, reader)
+	_ = writer.Close()
+	if err != nil {
+		if rmErr := os.Remove(newFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
+		}
+	}
+	return err
+}
+
+func downloadFile(ctx context.Context, url string) ([]byte, error) {
+	resp, err := fshttp.NewClient(ctx).Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(resp.Body, &err)
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed with %s downloading %s", resp.Status, url)
+	}
+	return ioutil.ReadAll(resp.Body)
+}

cmd/selfupdate/selfupdate_test.go

@@ -0,0 +1,198 @@
+package selfupdate
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/testy"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVersion(t *testing.T) {
+	testy.SkipUnreliable(t)
+
+	ctx := context.Background()
+
+	// a beta version can only have "v" prepended
+	resultVer, _, err := GetVersion(ctx, true, "1.2.3.4")
+	assert.NoError(t, err)
+	assert.Equal(t, "v1.2.3.4", resultVer)
+
+	// but a stable version syntax should be checked
+	_, _, err = GetVersion(ctx, false, "1")
+	assert.Error(t, err)
+	_, _, err = GetVersion(ctx, false, "1.")
+	assert.Error(t, err)
+	_, _, err = GetVersion(ctx, false, "1.2.")
+	assert.Error(t, err)
+	_, _, err = GetVersion(ctx, false, "1.2.3.4")
+	assert.Error(t, err)
+
+	// incomplete stable version should have micro release added
+	resultVer, _, err = GetVersion(ctx, false, "1.52")
+	assert.NoError(t, err)
+	assert.Equal(t, "v1.52.3", resultVer)
+}
+
+func makeTestDir() (testDir string, err error) {
+	const maxAttempts = 5
+	testDirBase := filepath.Join(os.TempDir(), "rclone-test-selfupdate.")
+
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		testDir = testDirBase + random.String(4)
+		err = os.MkdirAll(testDir, os.ModePerm)
+		if err == nil {
+			break
+		}
+	}
+	return
+}
+
+func TestInstallOnLinux(t *testing.T) {
+	testy.SkipUnreliable(t)
+	if runtime.GOOS != "linux" {
+		t.Skip("this is a Linux only test")
+	}
+
+	// Prepare for test
+	ctx := context.Background()
+	testDir, err := makeTestDir()
+	assert.NoError(t, err)
+	path := filepath.Join(testDir, "rclone")
+	defer func() {
+		_ = os.Chmod(path, 0644)
+		_ = os.RemoveAll(testDir)
+	}()
+
+	regexVer := regexp.MustCompile(`v[0-9]\S+`)
+
+	betaVer, _, err := GetVersion(ctx, true, "")
+	assert.NoError(t, err)
+
+	// Must do nothing if version isn't changing
+	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path, Version: fs.Version}))
+
+	// Must fail on non-writable file
+	assert.NoError(t, ioutil.WriteFile(path, []byte("test"), 0644))
+	assert.NoError(t, os.Chmod(path, 0000))
+	err = (InstallUpdate(ctx, &Options{Beta: true, Output: path}))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "run self-update as root")
+
+	// Must keep non-standard permissions
+	assert.NoError(t, os.Chmod(path, 0644))
+	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
+
+	info, err := os.Stat(path)
+	assert.NoError(t, err)
+	assert.Equal(t, os.FileMode(0644), info.Mode().Perm())
+
+	// Must remove temporary files
+	files, err := ioutil.ReadDir(testDir)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(files))
+
+	// Must contain valid executable
+	assert.NoError(t, os.Chmod(path, 0755))
+	cmd := exec.Command(path, "version")
+	output, err := cmd.CombinedOutput()
+	assert.NoError(t, err)
+	assert.True(t, cmd.ProcessState.Success())
+	assert.Equal(t, betaVer, regexVer.FindString(string(output)))
+}
+
+func TestRenameOnWindows(t *testing.T) {
+	testy.SkipUnreliable(t)
+	if runtime.GOOS != "windows" {
+		t.Skip("this is a Windows only test")
+	}
+
+	// Prepare for test
+	ctx := context.Background()
+
+	testDir, err := makeTestDir()
+	assert.NoError(t, err)
+	defer func() {
+		_ = os.RemoveAll(testDir)
+	}()
+
+	path := filepath.Join(testDir, "rclone.exe")
+	regexVer := regexp.MustCompile(`v[0-9]\S+`)
+
+	stableVer, _, err := GetVersion(ctx, false, "")
+	assert.NoError(t, err)
+
+	betaVer, _, err := GetVersion(ctx, true, "")
+	assert.NoError(t, err)
+
+	// Must not create temporary files when target doesn't exist
+	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
+
+	files, err := ioutil.ReadDir(testDir)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(files))
+
+	// Must save running executable as the "old" file
+	cmdWait := exec.Command(path, "config")
+	stdinWait, err := cmdWait.StdinPipe() // Make it run waiting for input
+	assert.NoError(t, err)
+	assert.NoError(t, cmdWait.Start())
+
+	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: false, Output: path}))
+	files, err = ioutil.ReadDir(testDir)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(files))
+
+	pathOld := filepath.Join(testDir, "rclone.old.exe")
+	_, err = os.Stat(pathOld)
+	assert.NoError(t, err)
+
+	cmd := exec.Command(path, "version")
+	output, err := cmd.CombinedOutput()
+	assert.NoError(t, err)
+	assert.True(t, cmd.ProcessState.Success())
+	assert.Equal(t, stableVer, regexVer.FindString(string(output)))
+
+	cmdOld := exec.Command(pathOld, "version")
+	output, err = cmdOld.CombinedOutput()
+	assert.NoError(t, err)
+	assert.True(t, cmdOld.ProcessState.Success())
+	assert.Equal(t, betaVer, regexVer.FindString(string(output)))
+
+	// Stop previous waiting executable, run new and saved executables
+	_ = stdinWait.Close()
+	_ = cmdWait.Wait()
+	time.Sleep(100 * time.Millisecond)
+
+	cmdWait = exec.Command(path, "config")
+	stdinWait, err = cmdWait.StdinPipe()
+	assert.NoError(t, err)
+	assert.NoError(t, cmdWait.Start())
+
+	cmdWaitOld := exec.Command(pathOld, "config")
+	stdinWaitOld, err := cmdWaitOld.StdinPipe()
+	assert.NoError(t, err)
+	assert.NoError(t, cmdWaitOld.Start())
+
+	// Updating when the "old" executable is running must produce a random "old" file
+	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
+	files, err = ioutil.ReadDir(testDir)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(files))
+
+	// Stop all waiting executables
+	_ = stdinWait.Close()
+	_ = cmdWait.Wait()
+	_ = stdinWaitOld.Close()
+	_ = cmdWaitOld.Wait()
+	time.Sleep(100 * time.Millisecond)
+}

cmd/selfupdate/verify.go

@@ -0,0 +1,72 @@
+package selfupdate
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/clearsign"
+)
+
+var ncwPublicKeyPGP = `-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGiBDuy3V0RBADVQOAF5aFiCxD3t2h6iAF2WMiaMlgZ6kX2i/u7addNkzX71VU9
+7NpI0SnsP5YWt+gEedST6OmFbtLfZWCR4KWn5XnNdjCMNhxaH6WccVqNm4ALPIqT
+59uVjkgf8RISmmoNJ1d+2wMWjQTUfwOEmoIgH6n+2MYNUKuctBrwAACflwCg1I1Q
+O/prv/5hczdpQCs+fL87DxsD/Rt7pIXvsIOZyQWbIhSvNpGalJuMkW5Jx92UjsE9
+1Ipo3Xr6SGRPgW9+NxAZAsiZfCX/19knAyNrN9blwL0rcPDnkhdGwK69kfjF+wq+
+QbogRGodbKhqY4v+cMNkKiemBuTQiWPkpKjifwNsD1fNjNKfDP3pJ64Yz7a4fuzV
+X1YwBACpKVuEen34lmcX6ziY4jq8rKibKBs4JjQCRO24kYoHDULVe+RS9krQWY5b
+e0foDhru4dsKccefK099G+WEzKVCKxupstWkTT/iJwajR8mIqd4AhD0wO9W3MCfV
+Ov8ykMDZ7qBWk1DHc87Ep3W1o8t8wq74ifV+HjhhWg8QAylXg7QlTmljayBDcmFp
+Zy1Xb29kIDxuaWNrQGNyYWlnLXdvb2QuY29tPohxBBMRCAAxBQsHCgMEAxUDAgMW
+AgECF4AWIQT79zfs6firGGBL0qyTk14C/ztU+gUCXjg2UgIZAQAKCRCTk14C/ztU
++lmmAJ4jH5FyULzStjisuTvHLTVz6G44eQCfaR5QGZFPseenE5ic2WeQcBcmtoG5
+Ag0EO7LdgRAIAI6QdFBg3/xa1gFKPYy1ihV9eSdGqwWZGJvokWsfCvHy5180tj/v
+UNOLAJrdqglMSvevNTXe8bT65D6423AAsLhch9wq/aNqrHolTYABzxRigjcS1//T
+yln5naGUzlVQXDVfrDk3Md/NrkdOFj7r/YyMF0+iWwpFz2qAjL95i5wfVZ1kWGrT
+2AmivE1wD1sWT/Ja3FDI0NRkU0Nbz/a0TKe4ml8iLVtZXpTRbxxCCPdkHXXgSyu1
+eZ4NrF/wTJuvwGn12TJ1EF95aVkHxAUw0+KmLGdcyBG+IKuHamrsjWIAXGXV///K
+AxPgUthccQ03HMjltFsrdmen5Q034YM3eOsAAwUH/jAKiIAA8LpZmZPnt9GZ4+Ol
+Zp22VAfyfDOFl4Ol+cWjkLAgjAFsm5gnOKcRSE/9XPxnQqkhw7+ZygYuUMgTDJ99
+/5IM1UQL3ooS+oFrDaE99S8bLeOe17skcdXcA/K83VqD9m93rQRnbtD+75zqKkZn
+9WNFyKCXg5P6PFPdNYRtlQKOcwFR9mHRLUmapQSAM8Y2pCgALZ7GViKQca8/TT1T
+gZk9fJMZYGez+IlOPxTJxjn80+vywk4/wdIWSiQj+8u5RzT9sjmm77wbMVNGRqYd
+W/EemW9Zz9vi0CIvJGgbPMqcuxw8e/5lnuQ6Mi3uDR0P2RNIAhFrdZpVSME8xQaI
+RgQYEQIABgUCO7LdgQAKCRCTk14C/ztU+mLBAKC2cdFy7eLaQAvyzcE2VK6HVIjn
+JACguA00bxLQuJ4+RCJrLFZP8ZlN2sc=
+=TtR5
+-----END PGP PUBLIC KEY BLOCK-----`
+
+func verifyHashsum(ctx context.Context, siteURL, version, archive string, hash []byte) error {
+	sumsURL := fmt.Sprintf("%s/%s/SHA256SUMS", siteURL, version)
+	sumsBuf, err := downloadFile(ctx, sumsURL)
+	if err != nil {
+		return err
+	}
+	fs.Debugf(nil, "downloaded hashsum list: %s", sumsURL)
+
+	keyRing, err := openpgp.ReadArmoredKeyRing(strings.NewReader(ncwPublicKeyPGP))
+	if err != nil {
+		return errors.New("unsupported signing key")
+	}
+	block, rest := clearsign.Decode(sumsBuf)
+	// block.Bytes = block.Bytes[1:] // uncomment to test invalid signature
+	_, err = openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body)
+	if err != nil || len(rest) > 0 {
+		return errors.New("invalid hashsum signature")
+	}
+
+	wantHash, err := findFileHash(sumsBuf, archive)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(hash, wantHash) {
+		return fmt.Errorf("archive hash mismatch: want %02x vs got %02x", wantHash, hash)
+	}
+	return nil
+}

cmd/selfupdate/writable_unix.go

@@ -0,0 +1,11 @@
+// +build !windows,!plan9,!js
+
+package selfupdate
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func writable(path string) bool {
+	return unix.Access(path, unix.W_OK) == nil
+}

cmd/selfupdate/writable_unsupported.go

@@ -0,0 +1,7 @@
+// +build plan9 js
+
+package selfupdate
+
+func writable(path string) bool {
+	return true
+}

cmd/selfupdate/writable_windows.go

@@ -0,0 +1,16 @@
+// +build windows
+
+package selfupdate
+
+import (
+	"os"
+)
+
+func writable(path string) bool {
+	info, err := os.Stat(path)
+	const UserWritableBit = 128
+	if err == nil {
+		return info.Mode().Perm()&UserWritableBit != 0
+	}
+	return false
+}

cmd/serve/dlna/dlna_test.go

@@ -13,12 +13,12 @@ import (
 
 	"github.com/anacrolix/dms/soap"
 
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/vfs"
 
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/cmd/serve/dlna/dlnaflags"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +41,7 @@ func startServer(t *testing.T, f fs.Fs) {
 }
 
 func TestInit(t *testing.T) {
-	config.LoadConfig(context.Background())
+	configfile.LoadConfig(context.Background())
 
 	f, err := fs.NewFs(context.Background(), "testdata/files")
 	l, _ := f.List(context.Background(), "")

cmd/serve/http/http_test.go

@@ -12,7 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -61,7 +61,7 @@ var (
 func TestInit(t *testing.T) {
 	ctx := context.Background()
 	// Configure the remote
-	config.LoadConfig(context.Background())
+	configfile.LoadConfig(context.Background())
 	// fs.Config.LogLevel = fs.LogLevelDebug
 	// fs.Config.DumpHeaders = true
 	// fs.Config.DumpBodies = true

cmd/serve/restic/restic_appendonly_test.go

@@ -1,6 +1,7 @@
 package restic
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"io"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/stretchr/testify/require"
 )
 
@@ -63,6 +65,8 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
 
 // TestResticHandler runs tests on the restic handler code, especially in append-only mode.
 func TestResticHandler(t *testing.T) {
+	ctx := context.Background()
+	configfile.LoadConfig(ctx)
 	buf := make([]byte, 32)
 	_, err := io.ReadFull(rand.Reader, buf)
 	require.NoError(t, err)

cmd/sync/sync.go

@@ -27,7 +27,8 @@ var commandDefinition = &cobra.Command{
 Sync the source to the destination, changing the destination
 only.  Doesn't transfer unchanged files, testing by size and
 modification time or MD5SUM.  Destination is updated to match
-source, including deleting files if necessary.
+source, including deleting files if necessary (except duplicate
+objects, see below).
 
 **Important**: Since this can cause data loss, test first with the
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
@@ -35,7 +36,8 @@ source, including deleting files if necessary.
     rclone sync -i SOURCE remote:DESTINATION
 
 Note that files in the destination won't be deleted if there were any
-errors at any point.
+errors at any point.  Duplicate objects (files with the same name, on
+those providers that support it) are also not yet handled.
 
 It is always the contents of the directory that is synced, not the
 directory so when source:path is a directory, it's the contents of
@@ -46,6 +48,9 @@ If dest:path doesn't exist, it is created and the source:path contents
 go there.
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
+
+**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
+See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)

cmd/test/histogram/histogram.go

@@ -0,0 +1,59 @@
+package histogram
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/test"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	test.Command.AddCommand(commandDefinition)
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "histogram [remote:path]",
+	Short: `Makes a histogram of file name characters.`,
+	Long: `This command outputs JSON which shows the histogram of characters used
+in filenames in the remote:path specified.
+
+The data doesn't contain any identifying information but is useful for
+the rclone developers when developing filename compression.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsDir(args)
+		ctx := context.Background()
+		ci := fs.GetConfig(ctx)
+		cmd.Run(false, false, command, func() error {
+			var hist [256]int64
+			err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+				for _, entry := range entries {
+					base := path.Base(entry.Remote())
+					for i := range base {
+						hist[base[i]]++
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+			enc := json.NewEncoder(os.Stdout)
+			// enc.SetIndent("", "\t")
+			err = enc.Encode(&hist)
+			if err != nil {
+				return err
+			}
+			fmt.Println()
+			return nil
+		})
+	},
+}

cmd/test/info/info.go

@@ -1,7 +1,7 @@
 package info
 
 // FIXME once translations are implemented will need a no-escape
-// option for Put so we can make these tests work agaig
+// option for Put so we can make these tests work again
 
 import (
 	"bytes"
@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path"
 	"regexp"
@@ -20,7 +21,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/info/internal"
+	"github.com/rclone/rclone/cmd/test"
+	"github.com/rclone/rclone/cmd/test/info/internal"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/hash"
@@ -35,6 +37,7 @@ var (
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
+	all                bool
 	uploadWait         time.Duration
 	positionLeftRe     = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
 	positionMiddleRe   = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
@@ -42,14 +45,15 @@ var (
 )
 
 func init() {
-	cmd.Root.AddCommand(commandDefinition)
+	test.Command.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file.")
-	flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
-	flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", true, "Check control characters.")
+	flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization.")
+	flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters.")
 	flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file.")
-	flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", true, "Check max filename length.")
-	flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", true, "Check uploads with indeterminate file size.")
+	flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length.")
+	flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size.")
+	flags.BoolVarP(cmdFlags, &all, "all", "", false, "Run all tests.")
 }
 
 var commandDefinition = &cobra.Command{
@@ -59,10 +63,20 @@ var commandDefinition = &cobra.Command{
 to write to the paths passed in and how long they can be.  It can take some
 time.  It will write test files into the remote:path passed in.  It outputs
 a bit of go code for each one.
+
+**NB** this can create undeletable files and other hazards - use with care
 `,
-	Hidden: true,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1e6, command, args)
+		if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !all {
+			log.Fatalf("no tests selected - select a test or use -all")
+		}
+		if all {
+			checkNormalization = true
+			checkControl = true
+			checkLength = true
+			checkStreaming = true
+		}
 		for i := range args {
 			f := cmd.NewFsDir(args[i : i+1])
 			cmd.Run(false, false, command, func() error {

cmd/test/info/internal/build_csv/main.go

@@ -11,7 +11,7 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/rclone/rclone/cmd/info/internal"
+	"github.com/rclone/rclone/cmd/test/info/internal"
 )
 
 func main() {

cmd/test/makefiles/makefiles.go

@@ -0,0 +1,144 @@
+// Package makefiles builds a directory structure with the required
+// number of files in of the required size.
+package makefiles
+
+import (
+	cryptrand "crypto/rand"
+	"io"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/test"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/spf13/cobra"
+)
+
+var (
+	// Flags
+	numberOfFiles            = 1000
+	averageFilesPerDirectory = 10
+	maxDepth                 = 10
+	minFileSize              = fs.SizeSuffix(0)
+	maxFileSize              = fs.SizeSuffix(100)
+	minFileNameLength        = 4
+	maxFileNameLength        = 12
+
+	// Globals
+	directoriesToCreate int
+	totalDirectories    int
+	fileNames           = map[string]struct{}{} // keep a note of which file name we've used already
+)
+
+func init() {
+	test.Command.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.IntVarP(cmdFlags, &numberOfFiles, "files", "", numberOfFiles, "Number of files to create")
+	flags.IntVarP(cmdFlags, &averageFilesPerDirectory, "files-per-directory", "", averageFilesPerDirectory, "Average number of files per directory")
+	flags.IntVarP(cmdFlags, &maxDepth, "max-depth", "", maxDepth, "Maximum depth of directory hierarchy")
+	flags.FVarP(cmdFlags, &minFileSize, "min-file-size", "", "Minimum size of file to create")
+	flags.FVarP(cmdFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
+	flags.IntVarP(cmdFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
+	flags.IntVarP(cmdFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "makefiles <dir>",
+	Short: `Make a random file hierarchy in <dir>`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		outputDirectory := args[0]
+		directoriesToCreate = numberOfFiles / averageFilesPerDirectory
+		averageSize := (minFileSize + maxFileSize) / 2
+		log.Printf("Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
+		root := &dir{name: outputDirectory, depth: 1}
+		for totalDirectories < directoriesToCreate {
+			root.createDirectories()
+		}
+		dirs := root.list("", []string{})
+		for i := 0; i < numberOfFiles; i++ {
+			dir := dirs[rand.Intn(len(dirs))]
+			writeFile(dir, fileName())
+		}
+		log.Printf("Done.")
+	},
+}
+
+// fileName creates a unique random file or directory name
+func fileName() (name string) {
+	for {
+		length := rand.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
+		name = random.String(length)
+		if _, found := fileNames[name]; !found {
+			break
+		}
+	}
+	fileNames[name] = struct{}{}
+	return name
+}
+
+// dir is a directory in the directory hierarchy being built up
+type dir struct {
+	name     string
+	depth    int
+	children []*dir
+	parent   *dir
+}
+
+// Create a random directory hierarchy under d
+func (d *dir) createDirectories() {
+	for totalDirectories < directoriesToCreate {
+		newDir := &dir{
+			name:   fileName(),
+			depth:  d.depth + 1,
+			parent: d,
+		}
+		d.children = append(d.children, newDir)
+		totalDirectories++
+		switch rand.Intn(4) {
+		case 0:
+			if d.depth < maxDepth {
+				newDir.createDirectories()
+			}
+		case 1:
+			return
+		}
+	}
+	return
+}
+
+// list the directory hierarchy
+func (d *dir) list(path string, output []string) []string {
+	dirPath := filepath.Join(path, d.name)
+	output = append(output, dirPath)
+	for _, subDir := range d.children {
+		output = subDir.list(dirPath, output)
+	}
+	return output
+}
+
+// writeFile writes a random file at dir/name
+func writeFile(dir, name string) {
+	err := os.MkdirAll(dir, 0777)
+	if err != nil {
+		log.Fatalf("Failed to make directory %q: %v", dir, err)
+	}
+	path := filepath.Join(dir, name)
+	fd, err := os.Create(path)
+	if err != nil {
+		log.Fatalf("Failed to open file %q: %v", path, err)
+	}
+	size := rand.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
+	_, err = io.CopyN(fd, cryptrand.Reader, size)
+	if err != nil {
+		log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
+	}
+	err = fd.Close()
+	if err != nil {
+		log.Fatalf("Failed to close file %q: %v", path, err)
+	}
+}

cmd/test/memory/memory.go

@@ -1,4 +1,4 @@
-package memtest
+package memory
 
 import (
 	"context"
@@ -6,19 +6,19 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/test"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	cmd.Root.AddCommand(commandDefinition)
+	test.Command.AddCommand(commandDefinition)
 }
 
 var commandDefinition = &cobra.Command{
-	Use:    "memtest remote:path",
-	Short:  `Load all the objects at remote:path and report memory stats.`,
-	Hidden: true,
+	Use:   "memory remote:path",
+	Short: `Load all the objects at remote:path into memory and report memory stats.`,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)

cmd/test/test.go

@@ -0,0 +1,27 @@
+package test
+
+import (
+	"github.com/rclone/rclone/cmd"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(Command)
+}
+
+// Command definition for cobra
+var Command = &cobra.Command{
+	Use:   "test <subcommand>",
+	Short: `Run a test command`,
+	Long: `Rclone test is used to run test commands.
+
+Select which test comand you want with the subcommand, eg
+
+    rclone test memory remote:
+
+Each subcommand has its own options which you can see in their help.
+
+**NB** Be careful running these commands, they may do strange things
+so reading their documentation first is recommended.
+`,
+}

cmd/version/version.go

@@ -59,7 +59,7 @@ Or
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		if check {
-			checkVersion()
+			CheckVersion()
 		} else {
 			cmd.ShowVersion()
 		}
@@ -74,8 +74,8 @@ func stripV(s string) string {
 	return s
 }
 
-// getVersion gets the version by checking the download repository passed in
-func getVersion(url string) (v *semver.Version, vs string, date time.Time, err error) {
+// GetVersion gets the version available for download
+func GetVersion(url string) (v *semver.Version, vs string, date time.Time, err error) {
 	resp, err := http.Get(url)
 	if err != nil {
 		return v, vs, date, err
@@ -101,9 +101,8 @@ func getVersion(url string) (v *semver.Version, vs string, date time.Time, err e
 	return v, vs, date, err
 }
 
-// check the current version against available versions
-func checkVersion() {
-	// Get Current version
+// CheckVersion checks the installed version against available downloads
+func CheckVersion() {
 	vCurrent, err := semver.NewVersion(stripV(fs.Version))
 	if err != nil {
 		fs.Errorf(nil, "Failed to parse version: %v", err)
@@ -111,7 +110,7 @@ func checkVersion() {
 	const timeFormat = "2006-01-02"
 
 	printVersion := func(what, url string) {
-		v, vs, t, err := getVersion(url + "version.txt")
+		v, vs, t, err := GetVersion(url + "version.txt")
 		if err != nil {
 			fs.Errorf(nil, "Failed to get rclone %s version: %v", what, err)
 			return

docs/content/authors.md

@@ -456,3 +456,22 @@ put them back in again.` >}}
   * Nicolas Rueff <nicolas@rueff.fr>
   * Pau Rodriguez-Estivill <prodrigestivill@gmail.com>
   * Bob Pusateri <BobPusateri@users.noreply.github.com>
+  * Alex JOST <25005220+dimejo@users.noreply.github.com>
+  * Alexey Tabakman <samosad.ru@gmail.com>
+  * David Sze <sze.david@gmail.com>
+  * cynthia kwok <cynthia.m.kwok@gmail.com>
+  * Ankur Gupta <agupta@egnyte.com>
+  * Miron Veryanskiy <MironVeryanskiy@gmail.com>
+  * K265 <k.265@qq.com>
+  * Vesnyx <Vesnyx@users.noreply.github.com>
+  * Dmitry Chepurovskiy <me@dm3ch.net>
+  * Rauno Ots <rauno.ots@cgi.com>
+  * Georg Neugschwandtner <georg.neugschwandtner@gmx.net>
+  * pvalls <polvallsrue@gmail.com>
+  * Robert Thomas <31854736+wolveix@users.noreply.github.com>
+  * Romeo Kienzler <romeo.kienzler@gmail.com>
+  * tYYGH <tYYGH@users.noreply.github.com>
+  * georne <77802995+georne@users.noreply.github.com>
+  * Maxwell Calman <mcalman@MacBook-Pro.local>
+  * Naveen Honest Raj <naveendurai19@gmail.com>
+  * Lucas Messenger <lmesseng@cisco.com>

docs/content/cache.md

@@ -205,7 +205,7 @@ These URLs are used by Plex internally to connect to the Plex server securely.
 
 The format for these URLs is the following:
 
-https://ip-with-dots-replaced.server-hash.plex.direct:32400/
+`https://ip-with-dots-replaced.server-hash.plex.direct:32400/`
 
 The `ip-with-dots-replaced` part can be any IPv4 address, where the dots
 have been replaced with dashes, e.g. `127.0.0.1` becomes `127-0-0-1`.

docs/content/changelog.md

@@ -5,6 +5,43 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.54.1 - 2021-03-08
+
+[See commits](https://github.com/rclone/rclone/compare/v1.54.0...v1.54.1)
+
+* Bug Fixes
+    * accounting: Fix --bwlimit when up or down is off (Nick Craig-Wood)
+    * docs
+        * Fix nesting of brackets and backticks in ftp docs (edwardxml)
+        * Fix broken link in sftp page (edwardxml)
+        * Fix typo in crypt.md (Romeo Kienzler)
+        * Changelog: Correct link to digitalis.io (Alex JOST)
+        * Replace #file-caching with #vfs-file-caching (Miron Veryanskiy)
+        * Convert bogus example link to code (edwardxml)
+        * Remove dead link from rc.md (edwardxml)
+    * rc: Sync,copy,move: document createEmptySrcDirs parameter (Nick Craig-Wood)
+    * lsjson: Fix unterminated JSON in the presence of errors (Nick Craig-Wood)
+* Mount
+    * Fix mount dropping on macOS by setting --daemon-timeout 10m (Nick Craig-Wood)
+* VFS
+    * Document simultaneous usage with the same cache shouldn't be used (Nick Craig-Wood)
+* B2
+    * Automatically raise upload cutoff to avoid spurious error (Nick Craig-Wood)
+    * Fix failed to create file system with application key limited to a prefix (Nick Craig-Wood)
+* Drive
+    * Refer to Shared Drives instead of Team Drives (Nick Craig-Wood)
+* Dropbox
+    * Add scopes to oauth request and optionally "members.read" (Nick Craig-Wood)
+* S3
+    * Fix failed to create file system with folder level permissions policy (Nick Craig-Wood)
+    * Fix Wasabi HEAD requests returning stale data by using only 1 transport (Nick Craig-Wood)
+    * Fix shared_credentials_file auth (Dmitry Chepurovskiy)
+    * Add --s3-no-head to reducing costs docs (Nick Craig-Wood)
+* Union
+    * Fix mkdir at root with remote:/ (Nick Craig-Wood)
+* Zoho
+    * Fix custom client id's (buengese)
+
 ## v1.54.0 - 2021-02-02
 
 [See commits](https://github.com/rclone/rclone/compare/v1.53.0...v1.54.0)

docs/content/chunker.md

@@ -151,6 +151,9 @@ Note that `list` assembles composite directory entries only when chunk names
 match the configured format and treats non-conforming file names as normal
 non-chunked files.
 
+When using `norename` transactions, chunk names will additionally have a unique
+file version suffix. For example, `BIG_FILE_NAME.rclone_chunk.001_bp562k`.
+
 
 ### Metadata
 
@@ -170,6 +173,7 @@ for composite files. Meta objects carry the following fields:
 - `nchunks` - number of data chunks in file
 - `md5`     - MD5 hashsum of composite file (if present)
 - `sha1`    - SHA1 hashsum (if present)
+- `txn`     - identifies current version of the file
 
 There is no field for composite file name as it's simply equal to the name
 of meta object on the wrapped remote. Please refer to respective sections
@@ -242,8 +246,8 @@ use modification time of the first data chunk.
 
 ### Migrations
 
-The idiomatic way to migrate to a different chunk size, hash type or
-chunk naming scheme is to:
+The idiomatic way to migrate to a different chunk size, hash type, transaction
+style or chunk naming scheme is to:
 
 - Collect all your chunked files under a directory and have your
   chunker remote point to it.
@@ -303,6 +307,8 @@ Chunker included in rclone releases up to `v1.54` can sometimes fail to
 detect metadata produced by recent versions of rclone. We recommend users
 to keep rclone up-to-date to avoid data corruption.
 
+Changing `transactions` is dangerous and requires explicit migration.
+
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/chunker/chunker.go then run make backenddocs" >}}
 ### Standard Options
 

docs/content/commands/rclone_mount.md

@@ -198,7 +198,7 @@ Without the use of `--vfs-cache-mode` this can only write files
 sequentially, it can only seek when reading.  This means that many
 applications won't work with their files on an rclone mount without
 `--vfs-cache-mode writes` or `--vfs-cache-mode full`.
-See the [File Caching](#file-caching) section for more info.
+See the [VFS File Caching](#vfs-file-caching) section for more info.
 
 The bucket based remotes (e.g. Swift, S3, Google Compute Storage, B2,
 Hubic) do not support the concept of empty directories, so empty
@@ -213,7 +213,7 @@ File systems expect things to be 100% reliable, whereas cloud storage
 systems are a long way from 100% reliable. The rclone sync/copy
 commands cope with this with lots of retries.  However rclone mount
 can't use retries in the same way without making local copies of the
-uploads. Look at the [file caching](#file-caching)
+uploads. Look at the [VFS File Caching](#vfs-file-caching)
 for solutions to make mount more reliable.
 
 ## Attribute caching

docs/content/crypt.md

@@ -82,7 +82,7 @@ as you would with any other remote, e.g. `rclone copy D:\docs secret:\docs`,
 and rclone will encrypt and decrypt as needed on the fly.
 If you access the wrapped remote `remote:path` directly you will bypass
 the encryption, and anything you read will be in encrypted form, and
-anything you write will be undencrypted. To avoid issues it is best to
+anything you write will be unencrypted. To avoid issues it is best to
 configure a dedicated path for encrypted content, and access it
 exclusively through a crypt remote.
 

docs/content/docs.md

@@ -161,8 +161,9 @@ The syntax of the paths passed to the rclone command are as follows.
 
 This refers to the local file system.
 
-On Windows only `\` may be used instead of `/` in local paths
-**only**, non local paths must use `/`.
+On Windows `\` may be used instead of `/` in local paths **only**,
+non local paths must use `/`. See [local filesystem](https://rclone.org/local/#windows-paths)
+documentation for more about Windows-specific paths.
 
 These paths needn't start with a leading `/` - if they don't then they
 will be relative to the current directory.
@@ -363,7 +364,7 @@ This option controls the bandwidth limit. For example
 
     --bwlimit 10M
     
-would mean limit the upload and download bandwidth to 10 Mbyte/s.
+would mean limit the upload and download bandwidth to 10 MByte/s.
 **NB** this is **bytes** per second not **bits** per second. To use a
 single limit, specify the desired bandwidth in kBytes/s, or use a
 suffix b|k|M|G. The default is `0` which means to not limit bandwidth.
@@ -373,7 +374,7 @@ The upload and download bandwidth can be specified seperately, as
 
     --bwlimit 10M:100k
 
-would mean limit the upload bandwidth to 10 Mbyte/s and the download
+would mean limit the upload bandwidth to 10 MByte/s and the download
 bandwidth to 100 kByte/s. Either limit can be "off" meaning no limit, so
 to just limit the upload bandwidth you would use
 
@@ -402,9 +403,9 @@ working hours could be:
 `--bwlimit "08:00,512k 12:00,10M 13:00,512k 18:00,30M 23:00,off"`
 
 In this example, the transfer bandwidth will be set to 512kBytes/sec
-at 8am every day. At noon, it will rise to 10Mbytes/s, and drop back
+at 8am every day. At noon, it will rise to 10MByte/s, and drop back
 to 512kBytes/sec at 1pm. At 6pm, the bandwidth limit will be set to
-30MBytes/s, and at 11pm it will be completely disabled (full speed).
+30MByte/s, and at 11pm it will be completely disabled (full speed).
 Anything between 11pm and 8am will remain unlimited.
 
 An example of timetable with `WEEKDAY` could be:
@@ -412,8 +413,8 @@ An example of timetable with `WEEKDAY` could be:
 `--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off"`
 
 It means that, the transfer bandwidth will be set to 512kBytes/sec on
-Monday. It will rise to 10Mbytes/s before the end of Friday. At 10:00
-on Saturday it will be set to 1Mbyte/s. From 20:00 on Sunday it will
+Monday. It will rise to 10MByte/s before the end of Friday. At 10:00
+on Saturday it will be set to 1MByte/s. From 20:00 on Sunday it will
 be unlimited.
 
 Timeslots without `WEEKDAY` are extended to the whole week. So this
@@ -553,6 +554,9 @@ location is for you.
 Use this flag to override the config location, e.g. `rclone
 --config=".myconfig" .config`.
 
+If this is set to empty string or the special value `/notfound` then
+rclone will keep the config file in memory only.
+
 ### --contimeout=TIME ###
 
 Set the connection timeout. This should be in go time format which
@@ -600,6 +604,21 @@ This flag can be useful for debugging and in exceptional circumstances
 (e.g. Google Drive limiting the total volume of Server Side Copies to
 100GB/day).
 
+### --dscp VALUE ###
+
+Specify a DSCP value or name to use in connections. This could help QoS
+system to identify traffic class. BE, EF, DF, LE, CSx and AFxx are allowed.
+
+See the description of [differentiated services](https://en.wikipedia.org/wiki/Differentiated_services) to get an idea of
+this field. Setting this to 1 (LE) to identify the flow to SCAVENGER class
+can avoid occupying too much bandwidth in a network with DiffServ support ([RFC 8622](https://tools.ietf.org/html/rfc8622)).
+
+For example, if you configured QoS on router to handle LE properly. Running:
+```
+rclone copy --dscp LE from:/from to:/to
+```
+would make the priority lower than usual internet flows.
+
 ### -n, --dry-run ###
 
 Do a trial run with no permanent changes.  Use this to see what rclone

docs/content/downloads.md

@@ -13,7 +13,7 @@ Rclone Download {{< version >}}
 | Intel/AMD - 32 Bit | {{< download windows 386 >}} | - | {{< download linux 386 >}} | {{< download linux 386 deb >}} | {{< download linux 386 rpm >}} | {{< download freebsd 386 >}} | {{< download netbsd 386 >}} | {{< download openbsd 386 >}} | {{< download plan9 386 >}} | - |
 | ARMv6 - 32 Bit | - | - | {{< download linux arm >}} | {{< download linux arm deb >}} | {{< download linux arm rpm >}} | {{< download freebsd arm >}} | {{< download netbsd arm >}} | - | - | - |
 | ARMv7 - 32 Bit | - | - | {{< download linux arm-v7 >}} | {{< download linux arm-v7 deb >}} | {{< download linux arm-v7 rpm >}} | {{< download freebsd arm-v7 >}} | {{< download netbsd arm-v7 >}} | - | - | - |
-| ARM - 64 Bit | - | - | {{< download linux arm64 >}} | {{< download linux arm64 deb >}} | {{< download linux arm64 rpm >}} | - | - | - | - | - |
+| ARM - 64 Bit | - | {{< download osx arm64 >}} | {{< download linux arm64 >}} | {{< download linux arm64 deb >}} | {{< download linux arm64 rpm >}} | - | - | - | - | - |
 | MIPS - Big Endian | - | - | {{< download linux mips >}} | {{< download linux mips deb >}} | {{< download linux mips rpm >}} | - | - | - | - | - |
 | MIPS - Little Endian | - | - | {{< download linux mipsle >}} | {{< download linux mipsle deb >}} | {{< download linux mipsle rpm >}} | - | - | - | - | - |
 
@@ -82,7 +82,7 @@ script) from a URL which doesn't change then you can use these links.
 | Intel/AMD - 32 Bit | {{< cdownload windows 386 >}} | - | {{< cdownload linux 386 >}} | {{< cdownload linux 386 deb >}} | {{< cdownload linux 386 rpm >}} | {{< cdownload freebsd 386 >}} | {{< cdownload netbsd 386 >}} | {{< cdownload openbsd 386 >}} | {{< cdownload plan9 386 >}} | - |
 | ARMv6 - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
 | ARMv7 - 32 Bit | - | - | {{< cdownload linux arm-v7 >}} | {{< cdownload linux arm-v7 deb >}} | {{< cdownload linux arm-v7 rpm >}} | {{< cdownload freebsd arm-v7 >}} | {{< cdownload netbsd arm-v7 >}} | - | - | - |
-| ARM - 64 Bit | - | - | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
+| ARM - 64 Bit | - | {{< cdownload osx arm64 >}} | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
 | MIPS - Big Endian | - | - | {{< cdownload linux mips >}} | {{< cdownload linux mips deb >}} | {{< cdownload linux mips rpm >}} | - | - | - | - | - |
 | MIPS - Little Endian | - | - | {{< cdownload linux mipsle >}} | {{< cdownload linux mipsle deb >}} | {{< cdownload linux mipsle rpm >}} | - | - | - | - | - |
 

docs/content/drive.md

@@ -72,7 +72,7 @@ If your browser doesn't open automatically go to the following link: http://127.
 Log in and authorize rclone for access
 Waiting for code...
 Got code
-Configure this as a team drive?
+Configure this as a Shared Drive (Team Drive)?
 y) Yes
 n) No
 y/n> n
@@ -279,23 +279,24 @@ Note: in case you configured a specific root folder on gdrive and rclone is unab
         `rclone -v foo@example.com lsf gdrive:backup`
 
 
-### Team drives ###
+### Shared drives (team drives) ###
 
-If you want to configure the remote to point to a Google Team Drive
-then answer `y` to the question `Configure this as a team drive?`.
+If you want to configure the remote to point to a Google Shared Drive
+(previously known as Team Drives) then answer `y` to the question
+`Configure this as a Shared Drive (Team Drive)?`.
 
-This will fetch the list of Team Drives from google and allow you to
-configure which one you want to use.  You can also type in a team
-drive ID if you prefer.
+This will fetch the list of Shared Drives from google and allow you to
+configure which one you want to use. You can also type in a Shared
+Drive ID if you prefer.
 
 For example:
 
 ```
-Configure this as a team drive?
+Configure this as a Shared Drive (Team Drive)?
 y) Yes
 n) No
 y/n> y
-Fetching team drive list...
+Fetching Shared Drive list...
 Choose a number from below, or type in your own value
  1 / Rclone Test
    \ "xxxxxxxxxxxxxxxxxxxx"
@@ -303,7 +304,7 @@ Choose a number from below, or type in your own value
    \ "yyyyyyyyyyyyyyyyyyyy"
  3 / Rclone Test 3
    \ "zzzzzzzzzzzzzzzzzzzz"
-Enter a Team Drive ID> 1
+Enter a Shared Drive ID> 1
 --------------------
 [remote]
 client_id =
@@ -674,7 +675,7 @@ Needed only if you want use SA instead of interactive login.
 
 #### --drive-team-drive
 
-ID of the Team Drive
+ID of the Shared Drive (Team Drive)
 
 - Config:      team_drive
 - Env Var:     RCLONE_DRIVE_TEAM_DRIVE
@@ -1137,11 +1138,11 @@ Options:
 
 #### drives
 
-List the shared drives available to this account
+List the Shared Drives available to this account
 
     rclone backend drives remote: [options] [<arguments>+]
 
-This command lists the shared drives (teamdrives) available to this
+This command lists the Shared Drives (Team Drives) available to this
 account.
 
 Usage:

docs/content/filtering.md

@@ -78,6 +78,24 @@ separator or the beginning of the path/file.
                - doesn't match "afile.jpg"
                - doesn't match "directory/file.jpg"
 
+The top level of the remote may not be the top level of the drive.
+
+E.g. for a Microsoft Windows local directory structure
+
+    F:
+    ├── bkp
+    ├── data
+    │   ├── excl
+    │   │   ├── 123.jpg
+    │   │   └── 456.jpg
+    │   ├── incl
+    │   │   └── document.pdf
+
+To copy the contents of folder `data` into folder `bkp` excluding the contents of subfolder
+`excl`the following command treats `F:\data` and `F:\bkp` as top level for filtering.
+
+`rclone copy F:\data\ F:\bkp\ --exclude=/excl/**`
+
 **Important** Use `/` in path/file name patterns and not `\` even if
 running on Microsoft Windows.
 
@@ -95,7 +113,7 @@ With `--ignore-case`
 
 ## How filter rules are applied to files
 
-Rclone path / file name filters are made up of one or more of the following flags:
+Rclone path/file name filters are made up of one or more of the following flags:
 
   * `--include`
   * `--include-from`
@@ -121,7 +139,7 @@ To mix up the order of processing includes and excludes use `--filter...`
 flags.
 
 Within `--include-from`, `--exclude-from` and `--filter-from` flags
-rules are processed from top to bottom of the referenced file..
+rules are processed from top to bottom of the referenced file.
 
 If there is an `--include` or `--include-from` flag specified, rclone
 implies a `- **` rule which it adds to the bottom of the internal rule
@@ -153,7 +171,7 @@ classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/
 
 ### How filter rules are applied to directories
 
-Rclone commands filter, and are applied to, path/file names not
+Rclone commands are applied to path/file names not
 directories. The entire contents of a directory can be matched
 to a filter by the pattern `directory/*` or recursively by
 `directory/**`.
@@ -167,15 +185,15 @@ recurse into subdirectories. This potentially optimises access to a remote
 by avoiding listing unnecessary directories. Whether optimisation is
 desirable depends on the specific filter rules and source remote content.
 
-Optimisation occurs if either:
+Directory recursion optimisation occurs if either:
 
-* A source remote does not support the rclone `ListR` primitive. `local`,
-`sftp`, `Microsoft OneDrive` and `WebDav` do not support `ListR`. Google
+* A source remote does not support the rclone `ListR` primitive. local,
+sftp, Microsoft OneDrive and WebDav do not support `ListR`. Google
 Drive and most bucket type storage do. [Full list](https://rclone.org/overview/#optional-features)
 
-* On other remotes, if the rclone command is not naturally recursive,
+* On other remotes (those that support `ListR`), if the rclone command is not naturally recursive, and
 provided it is not run with the `--fast-list` flag. `ls`, `lsf -R` and
-`size` are recursive but `sync`, `copy` and `move` are not.
+`size` are naturally recursive but `sync`, `copy` and `move` are not.
 
 * Whenever the `--disable ListR` flag is applied to an rclone command.
 
@@ -197,7 +215,7 @@ to be specified.
 
 E.g. `rclone ls remote: --include /directory/` will not match any
 files. Because it is an `--include` option the `--exclude **` rule
-is implied, and the `\directory\` pattern serves only to optimise
+is implied, and the `/directory/` pattern serves only to optimise
 access to the remote by ignoring everything outside of that directory.
 
 E.g. `rclone ls remote: --filter-from filter-list.txt` with a file
@@ -210,7 +228,7 @@ E.g. `rclone ls remote: --filter-from filter-list.txt` with a file
 
 All files in directories `dir1` or `dir2` or their subdirectories
 are completely excluded from the listing. Only files of suffix
-`'pdf` in the root of `remote:` or its subdirectories are listed.
+`pdf` in the root of `remote:` or its subdirectories are listed.
 The `- **` rule prevents listing of any path/files not previously
 matched by the rules above.
 
@@ -241,8 +259,8 @@ directories.
 
 E.g. on Microsoft Windows `rclone ls remote: --exclude "*\[{JP,KR,HK}\]*"`
 lists the files in `remote:` with `[JP]` or `[KR]` or `[HK]` in
-their name. The single quotes prevent the shell from interpreting the `\`
-characters. The `\` characters escape the `[` and `]` so ran clone filter
+their name. Quotes prevent the shell from interpreting the `\`
+characters.`\` characters escape the `[` and `]` so an rclone filter
 treats them literally rather than as a character-range. The `{` and `}`
 define an rclone pattern list. For other operating systems single quotes are
 required ie `rclone ls remote: --exclude '*\[{JP,KR,HK}\]*'`
@@ -489,13 +507,13 @@ The three files are transferred as follows:
     /home/user1/dir/ford → remote:backup/user1/dir/file
     /home/user2/prefect  → remote:backup/user2/stuff
 
-Alternatively if `/` is chosen as root `files-from.txt` would be:
+Alternatively if `/` is chosen as root `files-from.txt` will be:
 
     /home/user1/42
     /home/user1/dir/ford
     /home/user2/prefect
 
-The copy command would be:
+The copy command will be:
 
     rclone copy --files-from files-from.txt / remote:backup
 
@@ -576,10 +594,10 @@ Default units are seconds or the following abbreviations are valid:
 `--max-age` can also be specified as an absolute time in the following
 formats:
 
-- RFC3339 - e.g. "2006-01-02T15:04:05Z07:00"
-- ISO8601 Date and time, local timezone - "2006-01-02T15:04:05"
-- ISO8601 Date and time, local timezone - "2006-01-02 15:04:05"
-- ISO8601 Date - "2006-01-02" (YYYY-MM-DD)
+- RFC3339 - e.g. `2006-01-02T15:04:05Z` or `2006-01-02T15:04:05+07:00`
+- ISO8601 Date and time, local timezone - `2006-01-02T15:04:05`
+- ISO8601 Date and time, local timezone - `2006-01-02 15:04:05`
+- ISO8601 Date - `2006-01-02` (YYYY-MM-DD)
 
 `--max-age` applies only to files and not to directories.
 
@@ -603,7 +621,7 @@ old or more.
 **Important** this flag is dangerous to your data - use with `--dry-run`
 and `-v` first.
 
-In conjunction with `rclone sync` the `--delete-excluded deletes any files
+In conjunction with `rclone sync`, `--delete-excluded` deletes any files
 on the destination which are excluded from the command.
 
 E.g. the scope of `rclone sync -i A: B:` can be restricted:
@@ -643,7 +661,7 @@ not list `dir3`, `file3` or `.ignore`.
 ## Common pitfalls
 
 The most frequent filter support issues on
-the [rclone forum](https://https://forum.rclone.org/) are:
+the [rclone forum](https://forum.rclone.org/) are:
 
 * Not using paths relative to the root of the remote
 * Not using `/` to match from the root of a remote

docs/content/flags.md

@@ -42,6 +42,7 @@ These flags are available for every command.
       --dump DumpFlags                       List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles
       --dump-bodies                          Dump HTTP headers and bodies - may contain sensitive info
       --dump-headers                         Dump HTTP headers - may contain sensitive info
+      --dscp                                 DSCP Name or Value (default 0)
       --error-on-no-transfer                 Sets exit code 9 if no files are transferred, useful in scripts
       --exclude stringArray                  Exclude files matching pattern
       --exclude-from stringArray             Read exclude patterns from file (use - to read from stdin)

docs/content/ftp.md

@@ -109,8 +109,8 @@ excess files in the directory.
 
 Rlone FTP supports implicit FTP over TLS servers (FTPS). This has to
 be enabled in the FTP backend config for the remote, or with
-`[--ftp-tls]{#ftp-tls}`. The default FTPS port is `990`, not `21` and
-can be set with `[--ftp-port]{#ftp-port}`.
+[`--ftp-tls`](#ftp-tls). The default FTPS port is `990`, not `21` and
+can be set with [`--ftp-port`](#ftp-port).
 
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/ftp/ftp.go then run make backenddocs" >}}
 ### Standard Options

docs/content/install.md

@@ -18,6 +18,9 @@ See below for some expanded Linux / macOS instructions.
 See the [Usage section](/docs/#usage) of the docs for how to use rclone, or
 run `rclone -h`.
 
+Already installed rclone can be easily updated to the latest version
+using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
+
 ## Script installation ##
 
 To install rclone on Linux/macOS/BSD systems, run:

docs/content/local.md

@@ -10,15 +10,17 @@ Local paths are specified as normal filesystem paths, e.g. `/path/to/wherever`,
 
     rclone sync -i /home/source /tmp/destination
 
-Will sync `/home/source` to `/tmp/destination`
+Will sync `/home/source` to `/tmp/destination`.
 
-These can be configured into the config file for consistencies sake,
-but it is probably easier not to.
+For consistencies sake one can also configure a remote of type
+`local` in the config file, and access the local filesystem using
+rclone remote paths, e.g. `remote:path/to/wherever`, but it is probably
+easier not to.
 
 ### Modified time ###
 
 Rclone reads and writes the modified time using an accuracy determined by
-the OS.  Typically this is 1ns on Linux, 10 ns on Windows and 1 Second
+the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second
 on OS X.
 
 ### Filenames ###
@@ -109,15 +111,35 @@ These only get replaced if they are the last character in the name:
 Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
 as they can't be converted to UTF-16.
 
-### Long paths on Windows ###
+### Paths on Windows ###
 
-Rclone handles long paths automatically, by converting all paths to long
-[UNC paths](https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath)
-which allows paths up to 32,767 characters.
+On Windows there are many ways of specifying a path to a file system resource.
+Both absolute paths like `C:\path\to\wherever`, and relative paths like
+`..\wherever` can be used, and path separator can be either
+`\` (as in `C:\path\to\wherever`) or `/` (as in `C:/path/to/wherever`).
+Length of these paths are limited to 259 characters for files and 247
+characters for directories, but there is an alternative extended-length
+path format increasing the limit to (approximately) 32,767 characters.
+This format requires absolute paths and the use of prefix `\\?\`,
+e.g. `\\?\D:\some\very\long\path`. For convenience rclone will automatically
+convert regular paths into the corresponding extended-length paths,
+so in most cases you do not have to worry about this (read more [below](#long-paths)).
 
-This is why you will see that your paths, for instance `c:\files` is
-converted to the UNC path `\\?\c:\files` in the output,
-and `\\server\share` is converted to `\\?\UNC\server\share`.
+Note that Windows supports using the same prefix `\\?\` to
+specify path to volumes identified by their GUID, e.g.
+`\\?\Volume{b75e2c83-0000-0000-0000-602f00000000}\some\path`.
+This is *not* supported in rclone, due to an [issue](https://github.com/golang/go/issues/39785)
+in go.
+
+#### Long paths ####
+
+Rclone handles long paths automatically, by converting all paths to
+[extended-length path format](https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation), which allows paths up to 32,767 characters.
+
+This conversion will ensure paths are absolute and prefix them with
+the `\\?\`. This is why you will see that your paths, for instance
+`.\files` is shown as path `\\?\C:\files` in the output, and `\\server\share`
+as `\\?\UNC\server\share`.
 
 However, in rare cases this may cause problems with buggy file
 system drivers like [EncFS](https://github.com/rclone/rclone/issues/261).
@@ -141,7 +163,7 @@ And use rclone like this:
 
 This will use UNC paths on `c:\src` but not on `z:\dst`.
 Of course this will cause problems if the absolute path length of a
-file exceeds 258 characters on z, so only use this option if you have to.
+file exceeds 259 characters on z, so only use this option if you have to.
 
 ### Symlinks / Junction points
 
@@ -447,6 +469,21 @@ to override the default choice.
 - Type:        bool
 - Default:     false
 
+#### --local-no-preallocate
+
+Disable preallocation of disk space for transferred files
+
+Preallocation of disk space helps prevent filesystem fragmentation.
+However, some virtual filesystem layers (such as Google Drive File
+Stream) may incorrectly set the actual file size equal to the
+preallocated space, causing checksum and file size checks to fail.
+Use this flag to disable preallocation.
+
+- Config:      no_preallocate
+- Env Var:     RCLONE_LOCAL_NO_PREALLOCATE
+- Type:        bool
+- Default:     false
+
 #### --local-no-sparse
 
 Disable sparse files for multi-thread downloads