config: remove log.Fatal from loading config file #5234

This changes the Config interface so that it returns an error.

.github/ISSUE_TEMPLATE/Bug.md

@@ -5,19 +5,31 @@ about: Report a problem with rclone
 
 <!--
 
-Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
+We understand you are having a problem with rclone; we want to help you with that!
 
-If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
+**STOP and READ**
+**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
+Please show the effort you've put in to solving the problem and please be specific.
+People are volunteering their time to help! Low effort posts are not likely to get good answers!
+
+If you think you might have found a bug, try to replicate it with the latest beta (or stable).
+The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
+
+If you can still replicate it or just got a question then please use the rclone forum:
 
     https://forum.rclone.org/
 
-instead of filing an issue for a quick response.
+for a quick response instead of filing an issue on this repo.
 
-If you think you might have found a bug, please can you try to replicate it with the latest beta?
+If nothing else helps, then please fill in the info below which helps us help you.
 
-    https://beta.rclone.org/
-    
-If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
+**DO NOT REDACT** any information except passwords/keys/personal info.
+
+You should use 3 backticks to begin and end your paste to make it readable.
+
+Make sure to include a log obtained with '-vv'.
+
+You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
 
 Thank you
 
@@ -25,6 +37,11 @@ The Rclone Developers
 
 -->
 
+
+#### The associated forum post URL from `https://forum.rclone.org`
+
+
+
 #### What is the problem you are having with rclone?
 
 
@@ -37,7 +54,7 @@ The Rclone Developers
 
 
 
-####  Which cloud storage system are you using? (e.g. Google Drive)
+#### Which cloud storage system are you using? (e.g. Google Drive)
 
 
 

.github/ISSUE_TEMPLATE/Feature.md

@@ -7,12 +7,16 @@ about: Suggest a new feature or enhancement for rclone
 
 Welcome :-)
 
-So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
+So you've got an idea to improve rclone? We love that!
+You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
 
-Here is a checklist of things to do:
+Probably the latest beta (or stable) release has your feature, so try to update your rclone.
+The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
 
-  1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum first: https://forum.rclone.org/
+If it still isn't there, here is a checklist of things to do:
+
+  1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
+  2. Discuss on the forum: https://forum.rclone.org/
   3. Make a feature request issue (this is the right place!).
   4. Be prepared to get involved making the feature :-)
 
@@ -23,6 +27,10 @@ The Rclone Developers
 -->
 
 
+#### The associated forum post URL from `https://forum.rclone.org`
+
+
+
 #### What is your current rclone version (output from `rclone version`)?
 
 

.github/workflows/build.yml

@@ -213,50 +213,100 @@ jobs:
         # Deploy binaries if enabled in config && not a PR && not a fork
         if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
-  xgo:
-    timeout-minutes: 60
-    name: "xgo cross compile"
-    runs-on: ubuntu-latest
+  android:
+     timeout-minutes: 30
+     name: "android-all"
+     runs-on: ubuntu-latest
+     
+     steps:
+       - name: Checkout
+         uses: actions/checkout@v2
+         with:
+           fetch-depth: 0
 
-    steps:
+       # Upgrade together with NDK version
+       - name: Set up Go 1.14
+         uses: actions/setup-go@v1
+         with:
+           go-version: 1.14
 
-      - name: Checkout
-        uses: actions/checkout@v1
-        with:
-          # Checkout into a fixed path to avoid import path problems on go < 1.11
-          path: ./src/github.com/rclone/rclone
+       # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
+       - name: Force NDK version
+         run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;21.4.7075529" | grep -v = || true
 
-      - name: Set environment variables
-        shell: bash
-        run: |
-          echo 'GOPATH=${{ runner.workspace }}' >> $GITHUB_ENV
-          echo '${{ runner.workspace }}/bin' >> $GITHUB_PATH
+       - name: Go module cache
+         uses: actions/cache@v2
+         with:
+           path: ~/go/pkg/mod
+           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+           restore-keys: |
+             ${{ runner.os }}-go-
 
-      - name: Cross-compile rclone
-        run: |
-          docker pull billziss/xgo-cgofuse
-          GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
-          # xgo \
-          #     -image=billziss/xgo-cgofuse \
-          #     -targets=darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-          #     -tags cmount \
-          #     -dest build \
-          #     .
-          xgo \
-              -image=billziss/xgo-cgofuse \
-              -targets=android/*,ios/* \
-              -dest build \
-              .
+       - name: Set global environment variables
+         shell: bash
+         run: |
+           echo "VERSION=$(make version)" >> $GITHUB_ENV
 
-      - name: Build rclone
-        shell: bash
-        run: |
-          make
+       - name: build native rclone
+         run: |
+           make
 
-      - name: Upload artifacts
-        run: |
-          make ci_upload
-        env:
-          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-        # Upload artifacts if not a PR && not a fork
-        if: github.head_ref == '' && github.repository == 'rclone/rclone'
+       - name: arm-v7a Set environment variables
+         shell: bash
+         run: |
+           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
+           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
+           echo 'GOOS=android' >> $GITHUB_ENV
+           echo 'GOARCH=arm' >> $GITHUB_ENV
+           echo 'GOARM=7' >> $GITHUB_ENV
+           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
+           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+       - name: arm-v7a build
+         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
+
+       - name: arm64-v8a Set environment variables
+         shell: bash
+         run: |
+           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
+           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
+           echo 'GOOS=android' >> $GITHUB_ENV
+           echo 'GOARCH=arm64' >> $GITHUB_ENV
+           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
+           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+
+       - name: arm64-v8a build
+         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
+
+       - name: x86 Set environment variables
+         shell: bash
+         run: |
+           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
+           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
+           echo 'GOOS=android' >> $GITHUB_ENV
+           echo 'GOARCH=386' >> $GITHUB_ENV
+           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
+           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+
+       - name: x86 build
+         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
+
+       - name: x64 Set environment variables
+         shell: bash
+         run: |
+           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
+           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
+           echo 'GOOS=android' >> $GITHUB_ENV
+           echo 'GOARCH=amd64' >> $GITHUB_ENV
+           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
+           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+
+       - name: x64 build
+         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
+
+       - name: Upload artifacts
+         run: |
+           make ci_upload
+         env:
+           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
+         # Upload artifacts if not a PR && not a fork
+         if: github.head_ref == '' && github.repository == 'rclone/rclone'

.gitignore

@@ -1,6 +1,7 @@
 *~
 _junk/
 rclone
+rclone.exe
 build
 docs/public
 rclone.iml

CONTRIBUTING.md

@@ -33,10 +33,11 @@ page](https://github.com/rclone/rclone).
 
 Now in your terminal
 
-    go get -u github.com/rclone/rclone
-    cd $GOPATH/src/github.com/rclone/rclone
+    git clone https://github.com/rclone/rclone.git
+    cd rclone
     git remote rename origin upstream
     git remote add origin git@github.com:YOURUSER/rclone.git
+    go build
 
 Make a branch to add your new feature
 

Makefile

@@ -119,7 +119,7 @@ doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 rclone.1:	MANUAL.md
 	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
@@ -187,10 +187,10 @@ upload_github:
 	./bin/upload-github $(TAG)
 
 cross:	doc
-	go run bin/cross-compile.go -release current $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -release current $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 beta:
-	go run bin/cross-compile.go $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
@@ -198,7 +198,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -compile-only $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 ci_upload:
 	sudo chown -R $$USER build

RELEASE.md

@@ -76,6 +76,24 @@ Now
 The rclone docker image should autobuild on via GitHub actions.  If it doesn't
 or needs to be updated then rebuild like this.
 
+See: https://github.com/ilteoood/docker_buildx/issues/19
+See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
+
+```
+git co v1.54.1
+docker pull golang
+export DOCKER_CLI_EXPERIMENTAL=enabled
+docker buildx create --name actions_builder --use
+docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
+echo "Supported platforms: $SUPPORTED_PLATFORMS"
+docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
+docker buildx stop actions_builder
+```
+
+### Old build for linux/amd64 only
+
 ```
 docker pull golang
 docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .

VERSION

@@ -1 +1 @@
-v1.55.0
+v1.56.0

backend/alias/alias_internal_test.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local" // pull in test backend
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/stretchr/testify/require"
 )
 
@@ -19,7 +20,7 @@ var (
 )
 
 func prepare(t *testing.T, root string) {
-	config.LoadConfig(context.Background())
+	require.NoError(t, configfile.LoadConfig(context.Background()))
 
 	// Configure the remote
 	config.FileSet(remoteName, "type", "alias")

backend/all/all.go

@@ -41,6 +41,7 @@ import (
 	_ "github.com/rclone/rclone/backend/swift"
 	_ "github.com/rclone/rclone/backend/tardigrade"
 	_ "github.com/rclone/rclone/backend/union"
+	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"
 	_ "github.com/rclone/rclone/backend/yandex"
 	_ "github.com/rclone/rclone/backend/zoho"

backend/amazonclouddrive/amazonclouddrive.go

@@ -16,7 +16,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"path"
 	"strings"
@@ -70,11 +69,12 @@ func init() {
 		Prefix:      "acd",
 		Description: "Amazon Drive",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			err := oauthutil.Config(ctx, "amazon cloud drive", name, m, acdConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "checkpoint",
@@ -205,7 +205,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if resp != nil {
 		if resp.StatusCode == 401 {
 			f.tokenRenewer.Invalidate()
@@ -280,7 +283,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.getRootInfo()
+		_, err := f.getRootInfo(ctx)
 		return err
 	})
 
@@ -288,14 +291,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		_, resp, err = f.c.Account.GetEndpoints()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get endpoints")
 	}
 
 	// Get rootID
-	rootInfo, err := f.getRootInfo()
+	rootInfo, err := f.getRootInfo(ctx)
 	if err != nil || rootInfo.Id == nil {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
@@ -337,11 +340,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // getRootInfo gets the root folder info
-func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) {
+func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		rootInfo, resp, err = f.c.Nodes.GetRoot()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return rootInfo, err
 }
@@ -380,7 +383,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	var subFolder *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -407,7 +410,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var info *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -428,7 +431,7 @@ type listAllFn func(*acd.Node) bool
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	query := "parents:" + dirID
 	if directoriesOnly {
 		query += " AND kind:" + folderKind
@@ -449,7 +452,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 		var resp *http.Response
 		err = f.pacer.CallNoRetry(func() (bool, error) {
 			nodes, resp, err = f.c.Nodes.GetNodes(&opts)
-			return f.shouldRetry(resp, err)
+			return f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return false, err
@@ -508,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var iErr error
 	for tries := 1; tries <= maxTries; tries++ {
 		entries = nil
-		_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool {
 			remote := path.Join(dir, *node.Name)
 			switch *node.Kind {
 			case folderKind:
@@ -667,7 +670,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		if ok {
 			return false, nil
 		}
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -708,7 +711,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
+	err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
 	if err != nil {
 		return nil, err
 	}
@@ -803,7 +806,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var jsonStr string
 	err = srcFs.pacer.Call(func() (bool, error) {
 		jsonStr, err = srcInfo.GetMetadata()
-		return srcFs.shouldRetry(nil, err)
+		return srcFs.shouldRetry(ctx, nil, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error: error reading src metadata: %v", err)
@@ -815,7 +818,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return err
 	}
 
-	err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
+	err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
 	if err != nil {
 		return err
 	}
@@ -840,7 +843,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if check {
 		// check directory is empty
 		empty := true
-		_, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool {
 			switch *node.Kind {
 			case folderKind:
 				empty = false
@@ -865,7 +868,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = node.Trash()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -987,7 +990,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	var info *acd.File
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -1044,7 +1047,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		} else {
 			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	return in, err
 }
@@ -1067,7 +1070,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if ok {
 			return false, nil
 		}
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1077,70 +1080,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove a node
-func (f *Fs) removeNode(info *acd.Node) error {
+func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error {
 	var resp *http.Response
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = info.Trash()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return err
 }
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.removeNode(o.info)
+	return o.fs.removeNode(ctx, o.info)
 }
 
 // Restore a node
-func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
+func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Restore()
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return newInfo, err
 }
 
 // Changes name of given node
-func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
+func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return newInfo, err
 }
 
 // Replaces one parent with another, effectively moving the file. Leaves other
 // parents untouched. ReplaceParent cannot be used when the file is trashed.
-func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error {
+func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.ReplaceParent(oldParentID, newParentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // Adds one additional parent to object.
-func (f *Fs) addParent(info *acd.Node, newParentID string) error {
+func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.AddParent(newParentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // Remove given parent from object, leaving the other possible
 // parents untouched. Object can end up having no parents.
-func (f *Fs) removeParent(info *acd.Node, parentID string) error {
+func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.RemoveParent(parentID)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
 // moveNode moves the node given from the srcLeaf,srcDirectoryID to
 // the dstLeaf,dstDirectoryID
-func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
+func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
 	// fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID)
 	cantMove := fs.ErrorCantMove
 	if useDirErrorMsgs {
@@ -1154,7 +1157,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
 
 	if srcLeaf != dstLeaf {
 		// fs.Debugf(name, "renaming")
-		_, err = f.renameNode(srcInfo, dstLeaf)
+		_, err = f.renameNode(ctx, srcInfo, dstLeaf)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path rename failed: %v", err)
 			goto OnConflict
@@ -1162,7 +1165,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s
 	}
 	if srcDirectoryID != dstDirectoryID {
 		// fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID)
-		err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID)
+		err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path parent replace failed: %v", err)
 			return err
@@ -1175,13 +1178,13 @@ OnConflict:
 	fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.")
 
 	// fs.Debugf(name, "Trashing file")
-	err = f.removeNode(srcInfo)
+	err = f.removeNode(ctx, srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: remove node failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Renaming file")
-	_, err = f.renameNode(srcInfo, dstLeaf)
+	_, err = f.renameNode(ctx, srcInfo, dstLeaf)
 	if err != nil {
 		fs.Debugf(name, "Move: rename node failed: %v", err)
 		return err
@@ -1189,19 +1192,19 @@ OnConflict:
 	// note: replacing parent is forbidden by API, modifying them individually is
 	// okay though
 	// fs.Debugf(name, "Adding target parent")
-	err = f.addParent(srcInfo, dstDirectoryID)
+	err = f.addParent(ctx, srcInfo, dstDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: addParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "removing original parent")
-	err = f.removeParent(srcInfo, srcDirectoryID)
+	err = f.removeParent(ctx, srcInfo, srcDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: removeParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Restoring")
-	_, err = f.restoreNode(srcInfo)
+	_, err = f.restoreNode(ctx, srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: restoreNode node failed: %v", err)
 		return err

backend/azureblob/azureblob.go

@@ -217,6 +217,23 @@ This option controls how often unused buffers will be removed from the pool.`,
 				encoder.EncodeDel |
 				encoder.EncodeBackSlash |
 				encoder.EncodeRightPeriod),
+		}, {
+			Name:    "public_access",
+			Help:    "Public access level of a container: blob, container.",
+			Default: string(azblob.PublicAccessNone),
+			Examples: []fs.OptionExample{
+				{
+					Value: string(azblob.PublicAccessNone),
+					Help:  "The container and its blobs can be accessed only with an authorized request. It's a default value",
+				}, {
+					Value: string(azblob.PublicAccessBlob),
+					Help:  "Blob data within this container can be read via anonymous request.",
+				}, {
+					Value: string(azblob.PublicAccessContainer),
+					Help:  "Allow full public read access for container and blob data.",
+				},
+			},
+			Advanced: true,
 		}},
 	})
 }
@@ -241,6 +258,7 @@ type Options struct {
 	MemoryPoolFlushTime  fs.Duration          `config:"memory_pool_flush_time"`
 	MemoryPoolUseMmap    bool                 `config:"memory_pool_use_mmap"`
 	Enc                  encoder.MultiEncoder `config:"encoding"`
+	PublicAccess         string               `config:"public_access"`
 }
 
 // Fs represents a remote azure server
@@ -262,6 +280,7 @@ type Fs struct {
 	imdsPacer     *fs.Pacer                       // Same but for IMDS
 	uploadToken   *pacer.TokenDispenser           // control concurrency
 	pool          *pool.Pool                      // memory pool
+	publicAccess  azblob.PublicAccessType         // Container Public Access Level
 }
 
 // Object describes an azure object
@@ -335,6 +354,19 @@ func validateAccessTier(tier string) bool {
 	}
 }
 
+// validatePublicAccess checks if azureblob supports use supplied public access level
+func validatePublicAccess(publicAccess string) bool {
+	switch publicAccess {
+	case string(azblob.PublicAccessNone),
+		string(azblob.PublicAccessBlob),
+		string(azblob.PublicAccessContainer):
+		// valid cases
+		return true
+	default:
+		return false
+	}
+}
+
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	401, // Unauthorized (e.g. "Token has expired")
@@ -347,7 +379,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// FIXME interpret special errors - more to do here
 	if storageErr, ok := err.(azblob.StorageError); ok {
 		switch storageErr.ServiceCode() {
@@ -499,6 +534,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
 	}
 
+	if !validatePublicAccess((opt.PublicAccess)) {
+		return nil, errors.Errorf("Azure Blob: Supported public access level are %s and %s",
+			string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer))
+	}
+
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
 		name:        name,
@@ -517,6 +557,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			opt.MemoryPoolUseMmap,
 		),
 	}
+	f.publicAccess = azblob.PublicAccessType(opt.PublicAccess)
 	f.imdsPacer.SetRetries(5) // per IMDS documentation
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -578,7 +619,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// Retry as specified by the documentation:
 			// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
 			token, err = GetMSIToken(ctx, userMSI)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 
 		if err != nil {
@@ -594,7 +635,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			var refreshedToken adal.Token
 			err := f.imdsPacer.Call(func() (bool, error) {
 				refreshedToken, err = GetMSIToken(ctx, userMSI)
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			})
 			if err != nil {
 				// Failed to refresh.
@@ -803,7 +844,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 
 		if err != nil {
@@ -1029,7 +1070,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return err
@@ -1081,7 +1122,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 		}
 		// now try to create the container
 		return f.pacer.Call(func() (bool, error) {
-			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, f.publicAccess)
 			if err != nil {
 				if storageErr, ok := err.(azblob.StorageError); ok {
 					switch storageErr.ServiceCode() {
@@ -1098,7 +1139,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 					}
 				}
 			}
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 	}, nil)
 }
@@ -1136,10 +1177,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error {
 					return false, fs.ErrorDirNotFound
 				}
 
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			}
 
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 	})
 }
@@ -1212,7 +1253,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	err = f.pacer.Call(func() (bool, error) {
 		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1373,7 +1414,7 @@ func (o *Object) readMetaData() (err error) {
 	var blobProperties *azblob.BlobGetPropertiesResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
@@ -1408,7 +1449,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	blob := o.getBlobReference()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1451,7 +1492,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var downloadResponse *azblob.DownloadResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open for download")
@@ -1592,7 +1633,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Stream contents of the reader object to the given blob URL
 		blockBlobURL := blob.ToBlockBlobURL()
 		_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1620,7 +1661,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	ac := azblob.BlobAccessConditions{}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.Delete(ctx, snapShotOptions, ac)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 }
 
@@ -1649,7 +1690,7 @@ func (o *Object) SetTier(tier string) error {
 	ctx := context.Background()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 
 	if err != nil {

backend/b2/api/types.go

@@ -2,12 +2,11 @@ package api
 
 import (
 	"fmt"
-	"path"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/version"
 )
 
 // Error describes a B2 error response
@@ -63,16 +62,17 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-const versionFormat = "-v2006-01-02-150405.000"
+// HasVersion returns true if it looks like the passed filename has a timestamp on it.
+//
+// Note that the passed filename's timestamp may still be invalid even if this
+// function returns true.
+func HasVersion(remote string) bool {
+	return version.Match(remote)
+}
 
 // AddVersion adds the timestamp as a version string into the filename passed in.
 func (t Timestamp) AddVersion(remote string) string {
-	ext := path.Ext(remote)
-	base := remote[:len(remote)-len(ext)]
-	s := time.Time(t).Format(versionFormat)
-	// Replace the '.' with a '-'
-	s = strings.Replace(s, ".", "-", -1)
-	return base + s + ext
+	return version.Add(remote, time.Time(t))
 }
 
 // RemoveVersion removes the timestamp from a filename as a version string.
@@ -80,24 +80,9 @@ func (t Timestamp) AddVersion(remote string) string {
 // It returns the new file name and a timestamp, or the old filename
 // and a zero timestamp.
 func RemoveVersion(remote string) (t Timestamp, newRemote string) {
-	newRemote = remote
-	ext := path.Ext(remote)
-	base := remote[:len(remote)-len(ext)]
-	if len(base) < len(versionFormat) {
-		return
-	}
-	versionStart := len(base) - len(versionFormat)
-	// Check it ends in -xxx
-	if base[len(base)-4] != '-' {
-		return
-	}
-	// Replace with .xxx for parsing
-	base = base[:len(base)-4] + "." + base[len(base)-3:]
-	newT, err := time.Parse(versionFormat, base[versionStart:])
-	if err != nil {
-		return
-	}
-	return Timestamp(newT), base[:versionStart] + ext
+	time, newRemote := version.Remove(remote)
+	t = Timestamp(time)
+	return
 }
 
 // IsZero returns true if the timestamp is uninitialized

backend/b2/api/types_test.go

@@ -13,7 +13,6 @@ import (
 var (
 	emptyT api.Timestamp
 	t0     = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
-	t0r    = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
 	t1     = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
 )
 
@@ -36,40 +35,6 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
 	assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
 }
 
-func TestTimestampAddVersion(t *testing.T) {
-	for _, test := range []struct {
-		t        api.Timestamp
-		in       string
-		expected string
-	}{
-		{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
-		{t1, "potato", "potato-v2001-02-03-040506-123"},
-		{t1, "", "-v2001-02-03-040506-123"},
-	} {
-		actual := test.t.AddVersion(test.in)
-		assert.Equal(t, test.expected, actual, test.in)
-	}
-}
-
-func TestTimestampRemoveVersion(t *testing.T) {
-	for _, test := range []struct {
-		in             string
-		expectedT      api.Timestamp
-		expectedRemote string
-	}{
-		{"potato.txt", emptyT, "potato.txt"},
-		{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
-		{"potato-v2001-02-03-040506-123", t1, "potato"},
-		{"-v2001-02-03-040506-123", t1, ""},
-		{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
-		{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
-	} {
-		actualT, actualRemote := api.RemoveVersion(test.in)
-		assert.Equal(t, test.expectedT, actualT, test.in)
-		assert.Equal(t, test.expectedRemote, actualRemote, test.in)
-	}
-}
-
 func TestTimestampIsZero(t *testing.T) {
 	assert.True(t, emptyT.IsZero())
 	assert.False(t, t0.IsZero())

backend/b2/b2.go

@@ -305,7 +305,10 @@ var retryErrorCodes = []int{
 
 // shouldRetryNoAuth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// For 429 or 503 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.
@@ -336,7 +339,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 		}
 		return true, err
 	}
-	return f.shouldRetryNoReauth(resp, err)
+	return f.shouldRetryNoReauth(ctx, resp, err)
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -504,7 +507,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
-		return f.shouldRetryNoReauth(resp, err)
+		return f.shouldRetryNoReauth(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to authenticate")
@@ -1350,7 +1353,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
 	}
 	var request = api.GetDownloadAuthorizationRequest{
 		BucketID:               bucketID,
-		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
+		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.rootDirectory, remote)),
 		ValidDurationInSeconds: validDurationInSeconds,
 	}
 	var response api.GetDownloadAuthorizationResponse
@@ -1741,6 +1744,13 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		ContentType:     resp.Header.Get("Content-Type"),
 		Info:            Info,
 	}
+	// When reading files from B2 via cloudflare using
+	// --b2-download-url cloudflare strips the Content-Length
+	// headers (presumably so it can inject stuff) so use the old
+	// length read from the listing.
+	if info.Size < 0 {
+		info.Size = o.size
+	}
 	return resp, info, nil
 }
 

backend/box/api/types.go

@@ -36,13 +36,13 @@ func (t *Time) UnmarshalJSON(data []byte) error {
 
 // Error is returned from box when things go wrong
 type Error struct {
-	Type        string `json:"type"`
-	Status      int    `json:"status"`
-	Code        string `json:"code"`
-	ContextInfo json.RawMessage
-	HelpURL     string `json:"help_url"`
-	Message     string `json:"message"`
-	RequestID   string `json:"request_id"`
+	Type        string          `json:"type"`
+	Status      int             `json:"status"`
+	Code        string          `json:"code"`
+	ContextInfo json.RawMessage `json:"context_info"`
+	HelpURL     string          `json:"help_url"`
+	Message     string          `json:"message"`
+	RequestID   string          `json:"request_id"`
 }
 
 // Error returns a string for the error and satisfies the error interface
@@ -132,6 +132,38 @@ type UploadFile struct {
 	ContentModifiedAt Time   `json:"content_modified_at"`
 }
 
+// PreUploadCheck is the request for upload preflight check
+type PreUploadCheck struct {
+	Name   string `json:"name"`
+	Parent Parent `json:"parent"`
+	Size   *int64 `json:"size,omitempty"`
+}
+
+// PreUploadCheckResponse is the response from upload preflight check
+// if successful
+type PreUploadCheckResponse struct {
+	UploadToken string `json:"upload_token"`
+	UploadURL   string `json:"upload_url"`
+}
+
+// PreUploadCheckConflict is returned in the ContextInfo error field
+// from PreUploadCheck when the error code is "item_name_in_use"
+type PreUploadCheckConflict struct {
+	Conflicts struct {
+		Type        string `json:"type"`
+		ID          string `json:"id"`
+		FileVersion struct {
+			Type string `json:"type"`
+			ID   string `json:"id"`
+			Sha1 string `json:"sha1"`
+		} `json:"file_version"`
+		SequenceID string `json:"sequence_id"`
+		Etag       string `json:"etag"`
+		Sha1       string `json:"sha1"`
+		Name       string `json:"name"`
+	} `json:"conflicts"`
+}
+
 // UpdateFileModTime is used in Update File Info
 type UpdateFileModTime struct {
 	ContentModifiedAt Time `json:"content_modified_at"`

backend/box/box.go

@@ -17,7 +17,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -84,7 +83,7 @@ func init() {
 		Name:        "box",
 		Description: "Box",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			jsonFile, ok := m.Get("box_config_file")
 			boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 			boxAccessToken, boxAccessTokenOk := m.Get("access_token")
@@ -93,15 +92,16 @@ func init() {
 			if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
 				err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
 				if err != nil {
-					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
+					return errors.Wrap(err, "failed to configure token with jwt authentication")
 				}
 				// Else, if not using an access token, use oauth2
 			} else if boxAccessToken == "" || !boxAccessTokenOk {
 				err = oauthutil.Config(ctx, "box", name, m, oauthConfig, nil)
 				if err != nil {
-					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
+					return errors.Wrap(err, "failed to configure token with oauth authentication")
 				}
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "root_folder_id",
@@ -157,15 +157,15 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	jsonFile = env.ShellExpand(jsonFile)
 	boxConfig, err := getBoxConfig(jsonFile)
 	if err != nil {
-		log.Fatalf("Failed to configure token: %v", err)
+		return errors.Wrap(err, "get box config")
 	}
 	privateKey, err := getDecryptedPrivateKey(boxConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure token: %v", err)
+		return errors.Wrap(err, "get decrypted private key")
 	}
 	claims, err := getClaims(boxConfig, boxSubType)
 	if err != nil {
-		log.Fatalf("Failed to configure token: %v", err)
+		return errors.Wrap(err, "get claims")
 	}
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
@@ -317,10 +317,13 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	authRetry := false
 
-	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
+	if resp != nil && resp.StatusCode == 401 && strings.Contains(resp.Header.Get("Www-Authenticate"), "expired_token") {
 		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
@@ -548,7 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -585,7 +588,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -683,22 +686,80 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 	return o, leaf, directoryID, nil
 }
 
+// preUploadCheck checks to see if a file can be uploaded
+//
+// It returns "", nil if the file is good to go
+// It returns "ID", nil if the file must be updated
+func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
+	check := api.PreUploadCheck{
+		Name: f.opt.Enc.FromStandardName(leaf),
+		Parent: api.Parent{
+			ID: directoryID,
+		},
+	}
+	if size >= 0 {
+		check.Size = &size
+	}
+	opts := rest.Opts{
+		Method: "OPTIONS",
+		Path:   "/files/content/",
+	}
+	var result api.PreUploadCheckResponse
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, &check, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "item_name_in_use" {
+			var conflict api.PreUploadCheckConflict
+			err = json.Unmarshal(apiErr.ContextInfo, &conflict)
+			if err != nil {
+				return "", errors.Wrap(err, "pre-upload check: JSON decode failed")
+			}
+			if conflict.Conflicts.Type != api.ItemTypeFile {
+				return "", errors.Wrap(err, "pre-upload check: can't overwrite non file with file")
+			}
+			return conflict.Conflicts.ID, nil
+		}
+		return "", errors.Wrap(err, "pre-upload check")
+	}
+	return "", nil
+}
+
 // Put the object
 //
 // Copy the reader in to the new object which is returned
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
-	switch err {
-	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
-	case fs.ErrorObjectNotFound:
-		// Not found so create it
-		return f.PutUnchecked(ctx, in, src)
-	default:
+	// If directory doesn't exist, file doesn't exist so can upload
+	remote := src.Remote()
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
+	if err != nil {
+		if err == fs.ErrorDirNotFound {
+			return f.PutUnchecked(ctx, in, src, options...)
+		}
 		return nil, err
 	}
+
+	// Preflight check the upload, which returns the ID if the
+	// object already exists
+	ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
+	if err != nil {
+		return nil, err
+	}
+	if ID == "" {
+		return f.PutUnchecked(ctx, in, src, options...)
+	}
+
+	// If object exists then create a skeleton one with just id
+	o := &Object{
+		fs:     f,
+		remote: remote,
+		id:     ID,
+	}
+	return o, o.Update(ctx, in, src, options...)
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
@@ -740,7 +801,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -767,7 +828,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -839,7 +900,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -877,7 +938,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -895,7 +956,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read user info")
@@ -1008,7 +1069,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return info.SharedLink.URL, err
 }
@@ -1026,7 +1087,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1048,7 +1109,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list trash")
@@ -1182,7 +1243,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return info, err
 }
@@ -1215,7 +1276,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1255,7 +1316,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err

backend/box/upload.go

@@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return
 }
@@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -109,10 +109,10 @@ outer:
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
-				return shouldRetry(resp, err)
+				return shouldRetry(ctx, resp, err)
 			}
 			body, err = rest.ReadBody(resp)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		delay := defaultDelay
 		var why string
@@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }

backend/cache/cache_internal_test.go

@@ -836,7 +836,7 @@ func newRun() *run {
 	if uploadDir == "" {
 		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
 		if err != nil {
-			log.Fatalf("Failed to create temp dir: %v", err)
+			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
 	} else {
 		r.tmpUploadDir = uploadDir
@@ -892,7 +892,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.FileGet(remote, "type", "")
+		remoteType := config.FileGet(remote, "type")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -903,14 +903,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.FileGet(remote, "remote", "")
+			remoteRemote := config.FileGet(remote, "remote")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.FileGet(remoteWrapping, "type", "")
+			remoteType := config.FileGet(remoteWrapping, "type")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil

backend/chunker/chunker.go

@@ -277,13 +277,10 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		return nil, errors.New("can't point remote at itself - check the value of the remote setting")
 	}
 
-	baseName, basePath, err := fspath.Parse(remote)
+	baseName, basePath, err := fspath.SplitFs(remote)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
-	if baseName != "" {
-		baseName += ":"
-	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(basePath, rpath)
 	baseFs, err := cache.Get(ctx, baseName+remotePath)

backend/compress/compress.go

@@ -53,7 +53,7 @@ const (
 	Gzip         = 2
 )
 
-var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9+_]{11})$")
+var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
 
 // Register with Fs
 func init() {

backend/crypt/cipher.go

@@ -12,12 +12,14 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 	"unicode/utf8"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/crypto/scrypt"
@@ -442,11 +444,32 @@ func (c *Cipher) encryptFileName(in string) string {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
+
+		// Strip version string so that only the non-versioned part
+		// of the file name gets encrypted/obfuscated
+		hasVersion := false
+		var t time.Time
+		if i == (len(segments)-1) && version.Match(segments[i]) {
+			var s string
+			t, s = version.Remove(segments[i])
+			// version.Remove can fail, in which case it returns segments[i]
+			if s != segments[i] {
+				segments[i] = s
+				hasVersion = true
+			}
+		}
+
 		if c.mode == NameEncryptionStandard {
 			segments[i] = c.encryptSegment(segments[i])
 		} else {
 			segments[i] = c.obfuscateSegment(segments[i])
 		}
+
+		// Add back a version to the encrypted/obfuscated
+		// file name, if we stripped it off earlier
+		if hasVersion {
+			segments[i] = version.Add(segments[i], t)
+		}
 	}
 	return strings.Join(segments, "/")
 }
@@ -477,6 +500,21 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
+
+		// Strip version string so that only the non-versioned part
+		// of the file name gets decrypted/deobfuscated
+		hasVersion := false
+		var t time.Time
+		if i == (len(segments)-1) && version.Match(segments[i]) {
+			var s string
+			t, s = version.Remove(segments[i])
+			// version.Remove can fail, in which case it returns segments[i]
+			if s != segments[i] {
+				segments[i] = s
+				hasVersion = true
+			}
+		}
+
 		if c.mode == NameEncryptionStandard {
 			segments[i], err = c.decryptSegment(segments[i])
 		} else {
@@ -486,6 +524,12 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if err != nil {
 			return "", err
 		}
+
+		// Add back a version to the decrypted/deobfuscated
+		// file name, if we stripped it off earlier
+		if hasVersion {
+			segments[i] = version.Add(segments[i], t)
+		}
 	}
 	return strings.Join(segments, "/"), nil
 }
@@ -494,10 +538,18 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
 		remainingLength := len(in) - len(encryptedSuffix)
-		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
-			return in[:remainingLength], nil
+		if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
+			return "", ErrorNotAnEncryptedFile
 		}
-		return "", ErrorNotAnEncryptedFile
+		decrypted := in[:remainingLength]
+		if version.Match(decrypted) {
+			_, unversioned := version.Remove(decrypted)
+			if unversioned == "" {
+				return "", ErrorNotAnEncryptedFile
+			}
+		}
+		// Leave the version string on, if it was there
+		return decrypted, nil
 	}
 	return c.decryptFileName(in)
 }

backend/crypt/cipher_test.go

@@ -160,22 +160,29 @@ func TestEncryptFileName(t *testing.T) {
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Standard mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionStandard, "", "", false)
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
+	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Now off mode
 	c, _ = newCipher(NameEncryptionOff, "", "", true)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
+	assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
+	assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 	// Obfuscation mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
 	assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
+	assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 }
@@ -194,14 +201,19 @@ func TestDecryptFileName(t *testing.T) {
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
 		{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
+		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
 		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
 		{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
+		{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
+		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
+		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
 		{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
 		{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
 		{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
 		{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
+		{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
 	} {
 		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
 		actual, actualErr := c.DecryptFileName(test.in)

backend/crypt/crypt.go

@@ -404,13 +404,16 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read destination hash")
 		}
-		if srcHash != "" && dstHash != "" && srcHash != dstHash {
-			// remove object
-			err = o.Remove(ctx)
-			if err != nil {
-				fs.Errorf(o, "Failed to remove corrupted object: %v", err)
+		if srcHash != "" && dstHash != "" {
+			if srcHash != dstHash {
+				// remove object
+				err = o.Remove(ctx)
+				if err != nil {
+					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
+				}
+				return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
 			}
-			return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
+			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
 	}
 

backend/drive/drive.go

@@ -14,7 +14,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"mime"
 	"net/http"
 	"path"
@@ -183,13 +182,12 @@ func init() {
 		Description: "Google Drive",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			// Parse config into Options struct
 			opt := new(Options)
 			err := configstruct.Set(m, opt)
 			if err != nil {
-				fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
-				return
+				return errors.Wrap(err, "couldn't parse config into struct")
 			}
 
 			// Fill in the scopes
@@ -199,16 +197,17 @@ func init() {
 				m.Set("root_folder_id", "appDataFolder")
 			}
 
-			if opt.ServiceAccountFile == "" {
+			if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
 				err = oauthutil.Config(ctx, "drive", name, m, driveConfig, nil)
 				if err != nil {
-					log.Fatalf("Failed to configure token: %v", err)
+					return errors.Wrap(err, "failed to configure token")
 				}
 			}
 			err = configTeamDrive(ctx, opt, m, name)
 			if err != nil {
-				log.Fatalf("Failed to configure Shared Drive: %v", err)
+				return errors.Wrap(err, "failed to configure Shared Drive")
 			}
+			return nil
 		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
@@ -522,7 +521,7 @@ If this flag is set then rclone will ignore shortcut files completely.
 	} {
 		for mimeType, extension := range m {
 			if err := mime.AddExtensionType(extension, mimeType); err != nil {
-				log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
+				fs.Errorf("Failed to register MIME type %q: %v", mimeType, err)
 			}
 		}
 	}
@@ -590,13 +589,13 @@ type Fs struct {
 }
 
 type baseObject struct {
-	fs           *Fs    // what this object is part of
-	remote       string // The remote path
-	id           string // Drive Id of this object
-	modifiedDate string // RFC3339 time it was last modified
-	mimeType     string // The object MIME type
-	bytes        int64  // size of the object
-	parents      int    // number of parents
+	fs           *Fs      // what this object is part of
+	remote       string   // The remote path
+	id           string   // Drive Id of this object
+	modifiedDate string   // RFC3339 time it was last modified
+	mimeType     string   // The object MIME type
+	bytes        int64    // size of the object
+	parents      []string // IDs of the parent directories
 }
 type documentObject struct {
 	baseObject
@@ -641,7 +640,10 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err == nil {
 		return false, nil
 	}
@@ -695,20 +697,20 @@ func containsString(slice []string, s string) bool {
 }
 
 // getFile returns drive.File for the ID passed and fields passed in
-func (f *Fs) getFile(ID string, fields googleapi.Field) (info *drive.File, err error) {
+func (f *Fs) getFile(ctx context.Context, ID string, fields googleapi.Field) (info *drive.File, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Get(ID).
 			Fields(fields).
 			SupportsAllDrives(true).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	return info, err
 }
 
 // getRootID returns the canonical ID for the "root" ID
-func (f *Fs) getRootID() (string, error) {
-	info, err := f.getFile("root", "id")
+func (f *Fs) getRootID(ctx context.Context) (string, error) {
+	info, err := f.getFile(ctx, "root", "id")
 	if err != nil {
 		return "", errors.Wrap(err, "couldn't find root directory ID")
 	}
@@ -814,7 +816,7 @@ OUTER:
 		var files *drive.FileList
 		err = f.pacer.Call(func() (bool, error) {
 			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return false, errors.Wrap(err, "couldn't list directory")
@@ -837,7 +839,7 @@ OUTER:
 				if filesOnly && item.ShortcutDetails.TargetMimeType == driveFolderType {
 					continue
 				}
-				item, err = f.resolveShortcut(item)
+				item, err = f.resolveShortcut(ctx, item)
 				if err != nil {
 					return false, errors.Wrap(err, "list")
 				}
@@ -855,7 +857,7 @@ OUTER:
 				if !found {
 					continue
 				}
-				_, exportName, _, _ := f.findExportFormat(item)
+				_, exportName, _, _ := f.findExportFormat(ctx, item)
 				if exportName == "" || exportName != title {
 					continue
 				}
@@ -1155,7 +1157,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 		f.rootFolderID = f.opt.TeamDriveID
 	} else {
 		// otherwise look up the actual root ID
-		rootID, err := f.getRootID()
+		rootID, err := f.getRootID(ctx)
 		if err != nil {
 			if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
 				// 404 means that this scope does not have permission to get the
@@ -1236,7 +1238,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 		modifiedDate: modifiedDate,
 		mimeType:     info.MimeType,
 		bytes:        size,
-		parents:      len(info.Parents),
+		parents:      info.Parents,
 	}
 }
 
@@ -1328,26 +1330,26 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
 // newObjectWithInfo creates an fs.Object for any drive.File
 //
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
-func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
 	// If item has MD5 sum or a length it is a file stored on drive
 	if info.Md5Checksum != "" || info.Size > 0 {
 		return f.newRegularObject(remote, info), nil
 	}
 
-	extension, exportName, exportMimeType, isDocument := f.findExportFormat(info)
-	return f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
+	extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
+	return f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
 }
 
 // newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
 //
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
-	remote string, info *drive.File,
+	ctx context.Context, remote string, info *drive.File,
 	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
-	info, err = f.resolveShortcut(info)
+	info, err = f.resolveShortcut(ctx, info)
 	if err != nil {
 		return nil, errors.Wrap(err, "new object")
 	}
@@ -1395,7 +1397,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	}
 
 	remote = remote[:len(remote)-len(extension)]
-	obj, err := f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
+	obj, err := f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
 	switch {
 	case err != nil:
 		return nil, err
@@ -1412,7 +1414,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	pathID = actualID(pathID)
 	found, err = f.list(ctx, []string{pathID}, leaf, true, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
 		if !f.opt.SkipGdocs {
-			_, exportName, _, isDocument := f.findExportFormat(item)
+			_, exportName, _, isDocument := f.findExportFormat(ctx, item)
 			if exportName == leaf {
 				pathIDOut = item.Id
 				return true
@@ -1447,8 +1449,8 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		info, err = f.svc.Files.Create(createInfo).
 			Fields("id").
 			SupportsAllDrives(true).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return "", err
@@ -1483,15 +1485,15 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
-func (f *Fs) fetchFormats() {
+func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			about, err = f.svc.About.Get().
 				Fields("exportFormats,importFormats").
-				Do()
-			return f.shouldRetry(err)
+				Context(ctx).Do()
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			fs.Errorf(f, "Failed to get Drive exportFormats and importFormats: %v", err)
@@ -1508,8 +1510,8 @@ func (f *Fs) fetchFormats() {
 // if necessary.
 //
 // if the fetch fails then it will not export any drive formats
-func (f *Fs) exportFormats() map[string][]string {
-	f.fetchFormats()
+func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
+	f.fetchFormats(ctx)
 	return _exportFormats
 }
 
@@ -1517,8 +1519,8 @@ func (f *Fs) exportFormats() map[string][]string {
 // if necessary.
 //
 // if the fetch fails then it will not import any drive formats
-func (f *Fs) importFormats() map[string][]string {
-	f.fetchFormats()
+func (f *Fs) importFormats(ctx context.Context) map[string][]string {
+	f.fetchFormats(ctx)
 	return _importFormats
 }
 
@@ -1527,9 +1529,9 @@ func (f *Fs) importFormats() map[string][]string {
 //
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
-func (f *Fs) findExportFormatByMimeType(itemMimeType string) (
+func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
 	extension, mimeType string, isDocument bool) {
-	exportMimeTypes, isDocument := f.exportFormats()[itemMimeType]
+	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
 			_mimeType := mime.TypeByExtension(_extension)
@@ -1556,8 +1558,8 @@ func (f *Fs) findExportFormatByMimeType(itemMimeType string) (
 //
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", "", false)
-func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
-	extension, mimeType, isDocument = f.findExportFormatByMimeType(item.MimeType)
+func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension, filename, mimeType string, isDocument bool) {
+	extension, mimeType, isDocument = f.findExportFormatByMimeType(ctx, item.MimeType)
 	if extension != "" {
 		filename = item.Name + extension
 	}
@@ -1569,9 +1571,9 @@ func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType s
 // MIME type is returned
 //
 // When no match is found "" is returned.
-func (f *Fs) findImportFormat(mimeType string) string {
+func (f *Fs) findImportFormat(ctx context.Context, mimeType string) string {
 	mimeType = fixMimeType(mimeType)
-	ifs := f.importFormats()
+	ifs := f.importFormats(ctx)
 	for _, mt := range f.importMimeTypes {
 		if mt == mimeType {
 			importMimeTypes := ifs[mimeType]
@@ -1604,7 +1606,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	var iErr error
 	_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
-		entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item)
+		entry, err := f.itemToDirEntry(ctx, path.Join(dir, item.Name), item)
 		if err != nil {
 			iErr = err
 			return true
@@ -1717,7 +1719,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 					}
 				}
 				remote := path.Join(paths[i], item.Name)
-				entry, err := f.itemToDirEntry(remote, item)
+				entry, err := f.itemToDirEntry(ctx, remote, item)
 				if err != nil {
 					iErr = err
 					return true
@@ -1982,7 +1984,7 @@ func isShortcut(item *drive.File) bool {
 // Note that we assume shortcuts can't point to shortcuts. Google
 // drive web interface doesn't offer the option to create a shortcut
 // to a shortcut. The documentation is silent on the issue.
-func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error) {
+func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *drive.File, err error) {
 	if f.opt.SkipShortcuts || item.MimeType != shortcutMimeType {
 		return item, nil
 	}
@@ -1990,7 +1992,7 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 		fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
 		return item, nil
 	}
-	newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
+	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
 		if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
 			// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
@@ -2012,18 +2014,21 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 // itemToDirEntry converts a drive.File to an fs.DirEntry.
 // When the drive.File cannot be represented as an fs.DirEntry
 // (nil, nil) is returned.
-func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry, err error) {
+func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
 	switch {
 	case item.MimeType == driveFolderType:
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, item.Id)
 		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
 		d := fs.NewDir(remote, when).SetID(item.Id)
+		if len(item.Parents) > 0 {
+			d.SetParentID(item.Parents[0])
+		}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
 	default:
-		entry, err = f.newObjectWithInfo(remote, item)
+		entry, err = f.newObjectWithInfo(ctx, remote, item)
 		if err == fs.ErrorObjectNotFound {
 			return nil, nil
 		}
@@ -2090,12 +2095,12 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	importMimeType := ""
 
 	if f.importMimeTypes != nil && !f.opt.SkipGdocs {
-		importMimeType = f.findImportFormat(srcMimeType)
+		importMimeType = f.findImportFormat(ctx, srcMimeType)
 
 		if isInternalMimeType(importMimeType) {
 			remote = remote[:len(remote)-len(srcExt)]
 
-			exportExt, _, _ = f.findExportFormatByMimeType(importMimeType)
+			exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
 			if exportExt == "" {
 				return nil, errors.Errorf("No export format found for %q", importMimeType)
 			}
@@ -2125,8 +2130,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 				Fields(partialFields).
 				SupportsAllDrives(true).
 				KeepRevisionForever(f.opt.KeepRevisionForever).
-				Do()
-			return f.shouldRetry(err)
+				Context(ctx).Do()
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -2138,7 +2143,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 			return nil, err
 		}
 	}
-	return f.newObjectWithInfo(remote, info)
+	return f.newObjectWithInfo(ctx, remote, info)
 }
 
 // MergeDirs merges the contents of all the directories passed
@@ -2180,8 +2185,8 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 					AddParents(dstDir.ID()).
 					Fields("").
 					SupportsAllDrives(true).
-					Do()
-				return f.shouldRetry(err)
+					Context(ctx).Do()
+				return f.shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
@@ -2214,14 +2219,14 @@ func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 			_, err = f.svc.Files.Update(id, &info).
 				Fields("").
 				SupportsAllDrives(true).
-				Do()
+				Context(ctx).Do()
 		} else {
 			err = f.svc.Files.Delete(id).
 				Fields("").
 				SupportsAllDrives(true).
-				Do()
+				Context(ctx).Do()
 		}
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 }
 
@@ -2334,11 +2339,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	if isDoc {
 		// preserve the description on copy for docs
-		info, err := f.getFile(actualID(srcObj.id), "description")
+		info, err := f.getFile(ctx, actualID(srcObj.id), "description")
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to read description for Google Doc")
+			fs.Errorf(srcObj, "Failed to read description for Google Doc: %v", err)
+		} else {
+			createInfo.Description = info.Description
 		}
-		createInfo.Description = info.Description
 	} else {
 		// don't overwrite the description on copy for files
 		// this should work for docs but it doesn't - it is probably a bug in Google Drive
@@ -2354,13 +2360,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
-	newObject, err := f.newObjectWithInfo(remote, info)
+	newObject, err := f.newObjectWithInfo(ctx, remote, info)
 	if err != nil {
 		return nil, err
 	}
@@ -2454,7 +2460,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 
 	if err != nil {
@@ -2472,7 +2478,7 @@ func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
 	var td *drive.Drive
 	err = f.pacer.Call(func() (bool, error) {
 		td, err = f.svc.Drives.Get(f.opt.TeamDriveID).Fields("name,id,capabilities,createdTime,restrictions").Context(ctx).Do()
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get Shared Drive info")
@@ -2495,7 +2501,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		about, err = f.svc.About.Get().Fields("storageQuota").Context(ctx).Do()
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get Drive storageQuota")
@@ -2567,14 +2573,14 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			AddParents(dstParents).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	return f.newObjectWithInfo(remote, info)
+	return f.newObjectWithInfo(ctx, remote, info)
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
@@ -2604,8 +2610,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		_, err = f.svc.Permissions.Create(id, permission).
 			Fields("").
 			SupportsAllDrives(true).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return "", err
@@ -2647,8 +2653,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			AddParents(dstDirectoryID).
 			Fields("").
 			SupportsAllDrives(true).
-			Do()
-		return f.shouldRetry(err)
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -2666,7 +2672,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
 	go func() {
 		// get the StartPageToken early so all changes from now on get processed
-		startPageToken, err := f.changeNotifyStartPageToken()
+		startPageToken, err := f.changeNotifyStartPageToken(ctx)
 		if err != nil {
 			fs.Infof(f, "Failed to get StartPageToken: %s", err)
 		}
@@ -2691,7 +2697,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 				}
 			case <-tickerC:
 				if startPageToken == "" {
-					startPageToken, err = f.changeNotifyStartPageToken()
+					startPageToken, err = f.changeNotifyStartPageToken(ctx)
 					if err != nil {
 						fs.Infof(f, "Failed to get StartPageToken: %s", err)
 						continue
@@ -2706,15 +2712,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
-func (f *Fs) changeNotifyStartPageToken() (pageToken string, err error) {
+func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
 		changes := f.svc.Changes.GetStartPageToken().SupportsAllDrives(true)
 		if f.isTeamDrive {
 			changes.DriveId(f.opt.TeamDriveID)
 		}
-		startPageToken, err = changes.Do()
-		return f.shouldRetry(err)
+		startPageToken, err = changes.Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -2743,7 +2749,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 				changesCall.Spaces("appDataFolder")
 			}
 			changeList, err = changesCall.Context(ctx).Do()
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return
@@ -2939,8 +2945,8 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(dstFs.opt.KeepRevisionForever).
-			Do()
-		return dstFs.shouldRetry(err)
+			Context(ctx).Do()
+		return dstFs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "shortcut creation failed")
@@ -2948,24 +2954,24 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 	if isDir {
 		return nil, nil
 	}
-	return dstFs.newObjectWithInfo(dstPath, info)
+	return dstFs.newObjectWithInfo(ctx, dstPath, info)
 }
 
 // List all team drives
-func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err error) {
-	drives = []*drive.TeamDrive{}
-	listTeamDrives := f.svc.Teamdrives.List().PageSize(100)
+func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.Drive, err error) {
+	drives = []*drive.Drive{}
+	listTeamDrives := f.svc.Drives.List().PageSize(100)
 	var defaultFs Fs // default Fs with default Options
 	for {
-		var teamDrives *drive.TeamDriveList
+		var teamDrives *drive.DriveList
 		err = f.pacer.Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Context(ctx).Do()
-			return defaultFs.shouldRetry(err)
+			return defaultFs.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return drives, errors.Wrap(err, "listing Team Drives failed")
 		}
-		drives = append(drives, teamDrives.TeamDrives...)
+		drives = append(drives, teamDrives.Drives...)
 		if teamDrives.NextPageToken == "" {
 			break
 		}
@@ -3002,8 +3008,8 @@ func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurs
 				_, err := f.svc.Files.Update(item.Id, &update).
 					SupportsAllDrives(true).
 					Fields("trashed").
-					Do()
-				return f.shouldRetry(err)
+					Context(ctx).Do()
+				return f.shouldRetry(ctx, err)
 			})
 			if err != nil {
 				err = errors.Wrap(err, "failed to restore")
@@ -3045,7 +3051,7 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 
 // copy file with id to dest
 func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
-	info, err := f.getFile(id, f.fileFields)
+	info, err := f.getFile(ctx, id, f.fileFields)
 	if err != nil {
 		return errors.Wrap(err, "couldn't find id")
 	}
@@ -3053,7 +3059,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 		return errors.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
 	}
 	info.Name = f.opt.Enc.ToStandardName(info.Name)
-	o, err := f.newObjectWithInfo(info.Name, info)
+	o, err := f.newObjectWithInfo(ctx, info.Name, info)
 	if err != nil {
 		return err
 	}
@@ -3062,7 +3068,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 		return err
 	}
 	if destLeaf == "" {
-		destLeaf = info.Name
+		destLeaf = path.Base(o.Remote())
 	}
 	if destDir == "" {
 		destDir = "."
@@ -3354,7 +3360,7 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 
 	found, err := f.list(ctx, []string{directoryID}, leaf, false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
 		if !f.opt.SkipGdocs {
-			extension, exportName, exportMimeType, isDocument = f.findExportFormat(item)
+			extension, exportName, exportMimeType, isDocument = f.findExportFormat(ctx, item)
 			if exportName == leaf {
 				info = item
 				return true
@@ -3405,8 +3411,8 @@ func (o *baseObject) SetModTime(ctx context.Context, modTime time.Time) error {
 		info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			Do()
-		return o.fs.shouldRetry(err)
+			Context(ctx).Do()
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -3444,7 +3450,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return req, nil, err
@@ -3536,8 +3542,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			v2File, err = o.fs.v2Svc.Files.Get(actualID(o.id)).
 				Fields("downloadUrl").
 				SupportsAllDrives(true).
-				Do()
-			return o.fs.shouldRetry(err)
+				Context(ctx).Do()
+			return o.fs.shouldRetry(ctx, err)
 		})
 		if err == nil {
 			fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
@@ -3617,8 +3623,8 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 				Fields(partialFields).
 				SupportsAllDrives(true).
 				KeepRevisionForever(o.fs.opt.KeepRevisionForever).
-				Do()
-			return o.fs.shouldRetry(err)
+				Context(ctx).Do()
+			return o.fs.shouldRetry(ctx, err)
 		})
 		return
 	}
@@ -3661,7 +3667,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
+	newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
 	if err != nil {
 		return err
 	}
@@ -3685,7 +3691,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
 	if o.fs.importMimeTypes == nil || o.fs.opt.SkipGdocs {
 		return errors.Errorf("can't update google document type without --drive-import-formats")
 	}
-	importMimeType = o.fs.findImportFormat(updateInfo.MimeType)
+	importMimeType = o.fs.findImportFormat(ctx, updateInfo.MimeType)
 	if importMimeType == "" {
 		return errors.Errorf("no import format found for %q", srcMimeType)
 	}
@@ -3702,7 +3708,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
 	remote := src.Remote()
 	remote = remote[:len(remote)-o.extLen]
 
-	newO, err := o.fs.newObjectWithInfo(remote, info)
+	newO, err := o.fs.newObjectWithInfo(ctx, remote, info)
 	if err != nil {
 		return err
 	}
@@ -3722,7 +3728,7 @@ func (o *linkObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo
 
 // Remove an object
 func (o *baseObject) Remove(ctx context.Context) error {
-	if o.parents > 1 {
+	if len(o.parents) > 1 {
 		return errors.New("can't delete safely - has multiple parents")
 	}
 	return o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
@@ -3738,6 +3744,14 @@ func (o *baseObject) ID() string {
 	return o.id
 }
 
+// ParentID returns the ID of the Object parent if known, or "" if not
+func (o *baseObject) ParentID() string {
+	if len(o.parents) > 0 {
+		return o.parents[0]
+	}
+	return ""
+}
+
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
@@ -3798,10 +3812,13 @@ var (
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
+	_ fs.ParentIDer      = (*Object)(nil)
 	_ fs.Object          = (*documentObject)(nil)
 	_ fs.MimeTyper       = (*documentObject)(nil)
 	_ fs.IDer            = (*documentObject)(nil)
+	_ fs.ParentIDer      = (*documentObject)(nil)
 	_ fs.Object          = (*linkObject)(nil)
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
+	_ fs.ParentIDer      = (*linkObject)(nil)
 )

backend/drive/drive_internal_test.go

@@ -111,6 +111,7 @@ func TestInternalParseExtensions(t *testing.T) {
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
+	ctx := context.Background()
 	item := &drive.File{
 		Name:     "file",
 		MimeType: "application/vnd.google-apps.document",
@@ -128,7 +129,7 @@ func TestInternalFindExportFormat(t *testing.T) {
 	} {
 		f := new(Fs)
 		f.exportExtensions = test.extensions
-		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
+		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
 		assert.Equal(t, test.wantExtension, gotExtension)
 		if test.wantExtension != "" {
 			assert.Equal(t, item.Name+gotExtension, gotFilename)

backend/drive/upload.go

@@ -94,7 +94,7 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -202,7 +202,7 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
 			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
-			again, err := rx.f.shouldRetry(err)
+			again, err := rx.f.shouldRetry(ctx, err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/dropbox.go

@@ -25,7 +25,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"path"
 	"regexp"
 	"strings"
@@ -99,8 +98,10 @@ var (
 			"files.content.write",
 			"files.content.read",
 			"sharing.write",
+			"account_info.read", // needed for About
 			// "file_requests.write",
 			// "members.read", // needed for impersonate - but causes app to need to be approved by Dropbox Team Admin during the flow
+			// "team_data.member"
 		},
 		// Endpoint: oauth2.Endpoint{
 		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
@@ -130,8 +131,8 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config {
 	}
 	// Make a copy of the config
 	config := *dropboxConfig
-	// Make a copy of the scopes with "members.read" appended
-	config.Scopes = append(config.Scopes, "members.read")
+	// Make a copy of the scopes with extra scopes requires appended
+	config.Scopes = append(config.Scopes, "members.read", "team_data.member")
 	return &config
 }
 
@@ -142,7 +143,7 @@ func init() {
 		Name:        "dropbox",
 		Description: "Dropbox",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			opt := oauthutil.Options{
 				NoOffline: true,
 				OAuth2Opts: []oauth2.AuthCodeOption{
@@ -151,8 +152,9 @@ func init() {
 			}
 			err := oauthutil.Config(ctx, "dropbox", name, m, getOauthConfig(m), &opt)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "chunk_size",
@@ -292,7 +294,10 @@ func (f *Fs) Features() *fs.Features {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err == nil {
 		return false, err
 	}
@@ -307,7 +312,7 @@ func shouldRetry(err error) (bool, error) {
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
-			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
+			fs.Logf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
 			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
@@ -425,7 +430,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if f.root == "" {
 			return f, nil
 		}
-		_, err := f.findSharedFile(f.root)
+		_, err := f.findSharedFile(ctx, f.root)
 		f.root = ""
 		if err == nil {
 			return f, fs.ErrorIsFile
@@ -445,7 +450,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 
 		// root is not empty so we have find the right shared folder if it exists
-		id, err := f.findSharedFolder(dir)
+		id, err := f.findSharedFolder(ctx, dir)
 		if err != nil {
 			// if we didn't find the specified shared folder we have to bail out here
 			return nil, err
@@ -453,7 +458,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// we found the specified shared folder so let's mount it
 		// this will add it to the users normal root namespace and allows us
 		// to actually perform operations on it using the normal api endpoints.
-		err = f.mountSharedFolder(id)
+		err = f.mountSharedFolder(ctx, id)
 		if err != nil {
 			switch e := err.(type) {
 			case sharing.MountFolderAPIError:
@@ -477,7 +482,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
 			acc, err = f.users.GetCurrentAccount()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "get current account failed")
@@ -495,7 +500,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.setRoot(root)
 
 	// See if the root is actually an object
-	_, err = f.getFileMetadata(f.slashRoot)
+	_, err = f.getFileMetadata(ctx, f.slashRoot)
 	if err == nil {
 		newRoot := path.Dir(f.root)
 		if newRoot == "." {
@@ -529,12 +534,12 @@ func (f *Fs) setRoot(root string) {
 }
 
 // getMetadata gets the metadata for a file or directory
-func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
+func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		switch e := err.(type) {
@@ -549,8 +554,8 @@ func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool,
 }
 
 // getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
-	entry, notFound, err := f.getMetadata(filePath)
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
+	entry, notFound, err := f.getMetadata(ctx, filePath)
 	if err != nil {
 		return nil, err
 	}
@@ -565,8 +570,8 @@ func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err
 }
 
 // getDirMetadata gets the metadata for a directory
-func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
-	entry, notFound, err := f.getMetadata(dirPath)
+func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
+	entry, notFound, err := f.getMetadata(ctx, dirPath)
 	if err != nil {
 		return nil, err
 	}
@@ -583,7 +588,7 @@ func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -592,7 +597,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
 	if info != nil {
 		err = o.setMetadataFromEntry(info)
 	} else {
-		err = o.readEntryAndSetMetadata()
+		err = o.readEntryAndSetMetadata(ctx)
 	}
 	if err != nil {
 		return nil, err
@@ -604,14 +609,14 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if f.opt.SharedFiles {
-		return f.findSharedFile(remote)
+		return f.findSharedFile(ctx, remote)
 	}
-	return f.newObjectWithInfo(remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
 // listSharedFoldersApi lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
-func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
+func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
 	var res *sharing.ListFoldersResult
 	for {
@@ -621,7 +626,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListFolders(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, err
@@ -633,7 +638,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListFoldersContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -658,8 +663,8 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
 // findSharedFolder find the id for a given shared folder name
 // somewhat annoyingly there is no endpoint to query a shared folder by it's name
 // so our only option is to iterate over all shared folders
-func (f *Fs) findSharedFolder(name string) (id string, err error) {
-	entries, err := f.listSharedFolders()
+func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
+	entries, err := f.listSharedFolders(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -672,20 +677,20 @@ func (f *Fs) findSharedFolder(name string) (id string, err error) {
 }
 
 // mountSharedFolder mount a shared folder to the root namespace
-func (f *Fs) mountSharedFolder(id string) error {
+func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
 	arg := sharing.MountFolderArg{
 		SharedFolderId: id,
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.sharing.MountFolder(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
 
 // listReceivedFiles lists shared the user as access to (note this means individual
 // files not files contained in shared folders)
-func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
+func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
 	var res *sharing.ListFilesResult
 	for {
@@ -695,7 +700,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListReceivedFiles(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, err
@@ -707,7 +712,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 			}
 			err := f.pacer.Call(func() (bool, error) {
 				res, err = f.sharing.ListReceivedFilesContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -734,8 +739,8 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-func (f *Fs) findSharedFile(name string) (o *Object, err error) {
-	files, err := f.listReceivedFiles()
+func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
+	files, err := f.listReceivedFiles(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -758,10 +763,10 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) {
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	if f.opt.SharedFiles {
-		return f.listReceivedFiles()
+		return f.listReceivedFiles(ctx)
 	}
 	if f.opt.SharedFolders {
-		return f.listSharedFolders()
+		return f.listSharedFolders(ctx)
 	}
 
 	root := f.slashRoot
@@ -782,7 +787,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			err = f.pacer.Call(func() (bool, error) {
 				res, err = f.srv.ListFolder(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				switch e := err.(type) {
@@ -800,7 +805,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			err = f.pacer.Call(func() (bool, error) {
 				res, err = f.srv.ListFolderContinue(&arg)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return nil, errors.Wrap(err, "list continue")
@@ -830,7 +835,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
 				entries = append(entries, d)
 			} else if fileInfo != nil {
-				o, err := f.newObjectWithInfo(remote, fileInfo)
+				o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
 				if err != nil {
 					return nil, err
 				}
@@ -879,7 +884,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	}
 
 	// check directory doesn't exist
-	_, err := f.getDirMetadata(root)
+	_, err := f.getDirMetadata(ctx, root)
 	if err == nil {
 		return nil // directory exists already
 	} else if err != fs.ErrorDirNotFound {
@@ -896,7 +901,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.CreateFolderV2(&arg2)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
@@ -913,7 +918,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	if check {
 		// check directory exists
-		_, err = f.getDirMetadata(root)
+		_, err = f.getDirMetadata(ctx, root)
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
 		}
@@ -930,7 +935,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
 			res, err = f.srv.ListFolder(&arg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
@@ -943,7 +948,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
@@ -996,7 +1001,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.CopyV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "copy failed")
@@ -1057,7 +1062,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.MoveV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
@@ -1081,17 +1086,34 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
-		// FIXME this gives settings_error/not_authorized/.. errors
-		// and the expires setting isn't in the documentation so remove
-		// for now.
-		// Settings: &sharing.SharedLinkSettings{
-		// 	Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
-		// },
+		Settings: &sharing.SharedLinkSettings{
+			RequestedVisibility: &sharing.RequestedVisibility{
+				Tagged: dropbox.Tagged{Tag: sharing.RequestedVisibilityPublic},
+			},
+			Audience: &sharing.LinkAudience{
+				Tagged: dropbox.Tagged{Tag: sharing.LinkAudiencePublic},
+			},
+			Access: &sharing.RequestedLinkAccessLevel{
+				Tagged: dropbox.Tagged{Tag: sharing.RequestedLinkAccessLevelViewer},
+			},
+		},
 	}
+	if expire < fs.DurationOff {
+		expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
+		createArg.Settings.Expires = expiryTime
+	}
+	// FIXME note we can't set Settings for non enterprise dropbox
+	// because of https://github.com/dropbox/dropbox-sdk-go-unofficial/issues/75
+	// however this only goes wrong when we set Expires, so as a
+	// work-around remove Settings unless expire is set.
+	if expire == fs.DurationOff {
+		createArg.Settings = nil
+	}
+
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
 		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 
 	if err != nil && strings.Contains(err.Error(),
@@ -1104,7 +1126,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		var listRes *sharing.ListSharedLinksResult
 		err = f.pacer.Call(func() (bool, error) {
 			listRes, err = f.sharing.ListSharedLinks(&listArg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return
@@ -1146,7 +1168,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	dstPath := path.Join(f.slashRoot, dstRemote)
 
 	// Check if destination exists
-	_, err := f.getDirMetadata(dstPath)
+	_, err := f.getDirMetadata(ctx, dstPath)
 	if err == nil {
 		return fs.ErrorDirExists
 	} else if err != fs.ErrorDirNotFound {
@@ -1165,7 +1187,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.MoveV2(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "MoveDir failed")
@@ -1179,7 +1201,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var q *users.SpaceUsage
 	err = f.pacer.Call(func() (bool, error) {
 		q, err = f.users.GetSpaceUsage()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -1210,7 +1232,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
 	go func() {
 		// get the StartCursor early so all changes from now on get processed
-		startCursor, err := f.changeNotifyCursor()
+		startCursor, err := f.changeNotifyCursor(ctx)
 		if err != nil {
 			fs.Infof(f, "Failed to get StartCursor: %s", err)
 		}
@@ -1235,7 +1257,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 				}
 			case <-tickerC:
 				if startCursor == "" {
-					startCursor, err = f.changeNotifyCursor()
+					startCursor, err = f.changeNotifyCursor(ctx)
 					if err != nil {
 						fs.Infof(f, "Failed to get StartCursor: %s", err)
 						continue
@@ -1251,7 +1273,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 	}()
 }
 
-func (f *Fs) changeNotifyCursor() (cursor string, err error) {
+func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) {
 	var startCursor *files.ListFolderGetLatestCursorResult
 
 	err = f.pacer.Call(func() (bool, error) {
@@ -1266,7 +1288,7 @@ func (f *Fs) changeNotifyCursor() (cursor string, err error) {
 
 		startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
 
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -1296,7 +1318,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 		}
 
 		res, err = f.svc.ListFolderLongpoll(&args)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -1319,7 +1341,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			changeList, err = f.srv.ListFolderContinue(&arg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return "", errors.Wrap(err, "list continue")
@@ -1331,13 +1353,13 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			switch info := entry.(type) {
 			case *files.FolderMetadata:
 				entryType = fs.EntryDirectory
-				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
 			case *files.FileMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
 			case *files.DeletedMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
+				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
 			default:
 				fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
 				continue
@@ -1392,7 +1414,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != DbHashType {
 		return "", hash.ErrUnsupported
 	}
-	err := o.readMetaData()
+	err := o.readMetaData(ctx)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to read hash from metadata")
 	}
@@ -1416,17 +1438,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
 }
 
 // Reads the entry for a file from dropbox
-func (o *Object) readEntry() (*files.FileMetadata, error) {
-	return o.fs.getFileMetadata(o.remotePath())
+func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
+	return o.fs.getFileMetadata(ctx, o.remotePath())
 }
 
 // Read entry if not set and set metadata from it
-func (o *Object) readEntryAndSetMetadata() error {
+func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
 	// Last resort set time from client
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	entry, err := o.readEntry()
+	entry, err := o.readEntry(ctx)
 	if err != nil {
 		return err
 	}
@@ -1439,12 +1461,12 @@ func (o *Object) remotePath() string {
 }
 
 // readMetaData gets the info if it hasn't already been fetched
-func (o *Object) readMetaData() (err error) {
+func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if !o.modTime.IsZero() {
 		return nil
 	}
 	// Last resort
-	return o.readEntryAndSetMetadata()
+	return o.readEntryAndSetMetadata(ctx)
 }
 
 // ModTime returns the modification time of the object
@@ -1452,7 +1474,7 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
-	err := o.readMetaData()
+	err := o.readMetaData(ctx)
 	if err != nil {
 		fs.Debugf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1486,7 +1508,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		err = o.fs.pacer.Call(func() (bool, error) {
 			_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -1502,7 +1524,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, in, err = o.fs.srv.Download(&arg)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 
 	switch e := err.(type) {
@@ -1521,7 +1543,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // Will work optimally if size is >= uploadChunkSize. If the size is either
 // unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
 // avoidable request to the Dropbox API that does not carry payload.
-func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
+func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
 	chunkSize := int64(o.fs.opt.ChunkSize)
 	chunks := 0
 	if size != -1 {
@@ -1550,7 +1572,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 			return false, nil
 		}
 		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1676,11 +1698,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var err error
 	var entry *files.FileMetadata
 	if size > int64(o.fs.opt.ChunkSize) || size == -1 {
-		entry, err = o.uploadChunked(in, commitInfo, size)
+		entry, err = o.uploadChunked(ctx, in, commitInfo, size)
 	} else {
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			entry, err = o.fs.srv.Upload(commitInfo, in)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	}
 	if err != nil {
@@ -1698,7 +1720,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
 			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
 		})
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }

backend/fichier/api.go

@@ -28,7 +28,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// Detect this error which the integration tests provoke
 	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
 	//
@@ -74,7 +77,7 @@ func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
 	var file File
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't read file info")
@@ -96,7 +99,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
 	var token GetTokenResponse
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -124,7 +127,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
 	var sharedFiles SharedFolderResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -153,7 +156,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
 	filesList = &FilesList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -181,7 +184,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 	foldersList = &FoldersList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list folders")
@@ -275,7 +278,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons
 	response = &MakeFolderResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create folder")
@@ -302,7 +305,7 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't remove folder")
@@ -331,7 +334,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
 	response = &GenericOKResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -358,7 +361,7 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
 	response = &MoveFileResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -383,7 +386,7 @@ func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename stri
 	response = &CopyFileResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -405,7 +408,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 	response = &GetUploadNodeResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "didnt got an upload node")
@@ -448,7 +451,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -482,7 +485,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
 	response = &EndFileUploadResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {

backend/fichier/fichier.go

@@ -35,9 +35,7 @@ func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "fichier",
 		Description: "1Fichier",
-		Config: func(ctx context.Context, name string, config configmap.Mapper) {
-		},
-		NewFs: NewFs,
+		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
 			Name: "api_key",
@@ -348,8 +346,10 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		return nil, err
 	}
 
-	if len(fileUploadResponse.Links) != 1 {
-		return nil, errors.New("unexpected amount of files")
+	if len(fileUploadResponse.Links) == 0 {
+		return nil, errors.New("upload response not found")
+	} else if len(fileUploadResponse.Links) > 1 {
+		fs.Debugf(remote, "Multiple upload responses found, using the first")
 	}
 
 	link := fileUploadResponse.Links[0]

backend/fichier/object.go

@@ -94,7 +94,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.rest.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {

backend/filefabric/filefabric.go

@@ -228,7 +228,10 @@ var retryStatusCodes = []struct {
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err != nil {
 		return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 	}
@@ -401,7 +404,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr
 		// Refresh the body each retry
 		opts.Body = strings.NewReader(data.Encode())
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, result)
-		return f.shouldRetry(resp, err, result)
+		return f.shouldRetry(ctx, resp, err, result)
 	})
 	if err != nil {
 		return resp, err
@@ -1277,7 +1280,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader)
-		return o.fs.shouldRetry(resp, err, nil)
+		return o.fs.shouldRetry(ctx, resp, err, nil)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to upload")

backend/ftp/ftp.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -33,6 +34,12 @@ var (
 	currentUser = env.CurrentUser()
 )
 
+const (
+	minSleep      = 10 * time.Millisecond
+	maxSleep      = 2 * time.Second
+	decayConstant = 2 // bigger for slower decay, exponential
+)
+
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -104,6 +111,11 @@ given, rclone will empty the connection pool.
 Set to 0 to keep connections indefinitely.
 `,
 			Advanced: true,
+		}, {
+			Name:     "close_timeout",
+			Help:     "Maximum time to wait for a response to close.",
+			Default:  fs.Duration(60 * time.Second),
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -132,6 +144,7 @@ type Options struct {
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
+	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -151,6 +164,7 @@ type Fs struct {
 	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
 	tlsConf  *tls.Config
+	pacer    *fs.Pacer // pacer for FTP connections
 }
 
 // Object describes an FTP file
@@ -227,29 +241,41 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-type dialCtx struct {
-	f   *Fs
-	ctx context.Context
-}
-
-// dial a new connection with fshttp dialer
-func (d *dialCtx) dial(network, address string) (net.Conn, error) {
-	conn, err := fshttp.NewDialer(d.ctx).Dial(network, address)
-	if err != nil {
-		return nil, err
+// shouldRetry returns a boolean as to whether this err deserve to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
 	}
-	if d.f.tlsConf != nil {
-		conn = tls.Client(conn, d.f.tlsConf)
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusNotAvailable:
+			return true, err
+		}
 	}
-	return conn, err
+	return fserrors.ShouldRetry(err), err
 }
 
 // Open a new connection to the FTP server.
-func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
+func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
-	dCtx := dialCtx{f, ctx}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dCtx.dial)}
-	if f.opt.ExplicitTLS {
+
+	// Make ftp library dial with fshttp dialer optionally using TLS
+	dial := func(network, address string) (conn net.Conn, err error) {
+		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
+		if f.tlsConf != nil && err == nil {
+			conn = tls.Client(conn, f.tlsConf)
+		}
+		return
+	}
+	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
+
+	if f.opt.TLS {
+		// Our dialer takes care of TLS but ftp library also needs tlsConf
+		// as a trigger for sending PSBZ and PROT options to server.
+		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
+	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
 		// Initial connection needs to be cleartext for explicit TLS
 		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
@@ -267,18 +293,22 @@ func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
-	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
+	err = f.pacer.Call(func() (bool, error) {
+		c, err = ftp.Dial(f.dialAddr, ftpConfig...)
+		if err != nil {
+			return shouldRetry(ctx, err)
+		}
+		err = c.Login(f.user, f.pass)
+		if err != nil {
+			_ = c.Quit()
+			return shouldRetry(ctx, err)
+		}
+		return false, nil
+	})
 	if err != nil {
-		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
-		return nil, errors.Wrap(err, "ftpConnection Dial")
+		err = errors.Wrapf(err, "failed to make FTP connection to %q", f.dialAddr)
 	}
-	err = c.Login(f.user, f.pass)
-	if err != nil {
-		_ = c.Quit()
-		fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
-		return nil, errors.Wrap(err, "ftpConnection Login")
-	}
-	return c, nil
+	return c, err
 }
 
 // Get an FTP connection from the pool, or open a new one
@@ -411,6 +441,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		dialAddr: dialAddr,
 		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 		tlsConf:  tlsConfig,
+		pacer:    fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -931,8 +962,8 @@ func (f *ftpReadCloser) Close() error {
 	go func() {
 		errchan <- f.rc.Close()
 	}()
-	// Wait for Close for up to 60 seconds
-	timer := time.NewTimer(60 * time.Second)
+	// Wait for Close for up to 60 seconds by default
+	timer := time.NewTimer(time.Duration(f.f.opt.CloseTimeout))
 	select {
 	case err = <-errchan:
 		timer.Stop()

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"path"
 	"strings"
@@ -76,17 +75,18 @@ func init() {
 		Prefix:      "gcs",
 		Description: "Google Cloud Storage (this is not Google Drive)",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
 			if saFile != "" || saCreds != "" || anonymous == "true" {
-				return
+				return nil
 			}
 			err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "project_number",
@@ -329,7 +329,10 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func shouldRetry(err error) (again bool, errOut error) {
+func shouldRetry(ctx context.Context, err error) (again bool, errOut error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	again = false
 	if err != nil {
 		if fserrors.ShouldRetry(err) {
@@ -455,7 +458,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			newRoot := path.Dir(f.root)
@@ -521,7 +524,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
 			objects, err = list.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			if gErr, ok := err.(*googleapi.Error); ok {
@@ -624,7 +627,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
 			buckets, err = listBuckets.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -750,7 +753,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			// Bucket already exists
@@ -785,7 +788,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
 			_, err = insertBucket.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	}, nil)
 }
@@ -802,7 +805,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
 			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	})
 }
@@ -848,7 +851,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	for {
 		err = f.pacer.Call(func() (bool, error) {
 			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, err
@@ -941,7 +944,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
@@ -1012,7 +1015,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = copyObject.Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1043,7 +1046,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1109,7 +1112,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = insertObject.Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1124,7 +1127,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }

backend/googlephotos/googlephotos.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	golog "log"
 	"net/http"
 	"net/url"
 	"path"
@@ -78,13 +77,12 @@ func init() {
 		Prefix:      "gphotos",
 		Description: "Google Photos",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			// Parse config into Options struct
 			opt := new(Options)
 			err := configstruct.Set(m, opt)
 			if err != nil {
-				fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
-				return
+				return errors.Wrap(err, "couldn't parse config into struct")
 			}
 
 			// Fill in the scopes
@@ -97,7 +95,7 @@ func init() {
 			// Do the oauth
 			err = oauthutil.Config(ctx, "google photos", name, m, oauthConfig, nil)
 			if err != nil {
-				golog.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
 
 			// Warn the user
@@ -108,6 +106,7 @@ func init() {
 
 `)
 
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "read_only",
@@ -240,7 +239,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -329,7 +331,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
 	var openIDconfig map[string]interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "couldn't read openID config")
@@ -358,7 +360,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't read user info")
@@ -389,7 +391,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 	var res interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't revoke token")
@@ -476,7 +478,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't list albums")
@@ -531,7 +533,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list files")
@@ -675,7 +677,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create album")
@@ -810,7 +812,7 @@ func (o *Object) Size() int64 {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		fs.Debugf(o, "Reading size failed: %v", err)
@@ -861,7 +863,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't get media item")
@@ -938,7 +940,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -993,10 +995,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		token, err = rest.ReadBody(resp)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't upload file")
@@ -1024,7 +1026,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.BatchCreateResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create media item")
@@ -1069,7 +1071,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't delete item from album")

backend/hdfs/object.go

@@ -109,7 +109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	dirname := path.Dir(realpath)
 	fs.Debugf(o.fs, "update [%s]", realpath)
 
-	err := o.fs.client.MkdirAll(dirname, 755)
+	err := o.fs.client.MkdirAll(dirname, 0755)
 	if err != nil {
 		return err
 	}

backend/http/http_internal_test.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/rest"
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	ts := httptest.NewServer(handler)
 
 	// Configure the remote
-	config.LoadConfig(context.Background())
+	require.NoError(t, configfile.LoadConfig(context.Background()))
 	// fs.Config.LogLevel = fs.LogLevelDebug
 	// fs.Config.DumpHeaders = true
 	// fs.Config.DumpBodies = true

backend/hubic/hubic.go

@@ -11,7 +11,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"strings"
 	"time"
@@ -56,11 +55,12 @@ func init() {
 		Name:        "hubic",
 		Description: "Hubic",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			err := oauthutil.Config(ctx, "hubic", name, m, oauthConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
 	})

backend/jottacloud/jottacloud.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -87,12 +86,12 @@ func init() {
 		Name:        "jottacloud",
 		Description: "Jottacloud",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			refresh := false
 			if version, ok := m.Get("configVersion"); ok {
 				ver, err := strconv.Atoi(version)
 				if err != nil {
-					log.Fatalf("Failed to parse config version - corrupted config")
+					return errors.Wrap(err, "failed to parse config version - corrupted config")
 				}
 				refresh = (ver != configVersion) && (ver != v1configVersion)
 			}
@@ -104,7 +103,7 @@ func init() {
 				if ok && tokenString != "" {
 					fmt.Printf("Already have a token - refresh?\n")
 					if !config.Confirm(false) {
-						return
+						return nil
 					}
 				}
 			}
@@ -116,11 +115,13 @@ func init() {
 
 			switch config.ChooseNumber("Your choice", 1, 3) {
 			case 1:
-				v2config(ctx, name, m)
+				return v2config(ctx, name, m)
 			case 2:
-				v1config(ctx, name, m)
+				return v1config(ctx, name, m)
 			case 3:
-				teliaCloudConfig(ctx, name, m)
+				return teliaCloudConfig(ctx, name, m)
+			default:
+				return errors.New("unknown config choice")
 			}
 		},
 		Options: []fs.Option{{
@@ -235,11 +236,14 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
+func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) error {
 	teliaCloudOauthConfig := &oauth2.Config{
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  teliaCloudAuthURL,
@@ -252,15 +256,14 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
 
 	err := oauthutil.Config(ctx, "jottacloud", name, m, teliaCloudOauthConfig, nil)
 	if err != nil {
-		log.Fatalf("Failed to configure token: %v", err)
-		return
+		return errors.Wrap(err, "failed to configure token")
 	}
 
 	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
 	if config.Confirm(false) {
 		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, teliaCloudOauthConfig)
 		if err != nil {
-			log.Fatalf("Failed to load oAuthClient: %s", err)
+			return errors.Wrap(err, "failed to load oAuthClient")
 		}
 
 		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
@@ -268,7 +271,7 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
 
 		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
 		if err != nil {
-			log.Fatalf("Failed to setup mountpoint: %s", err)
+			return errors.Wrap(err, "failed to setup mountpoint")
 		}
 		m.Set(configDevice, device)
 		m.Set(configMountpoint, mountpoint)
@@ -277,17 +280,18 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
 	m.Set("configVersion", strconv.Itoa(configVersion))
 	m.Set(configClientID, teliaCloudClientID)
 	m.Set(configTokenURL, teliaCloudTokenURL)
+	return nil
 }
 
 // v1config configure a jottacloud backend using legacy authentication
-func v1config(ctx context.Context, name string, m configmap.Mapper) {
+func v1config(ctx context.Context, name string, m configmap.Mapper) error {
 	srv := rest.NewClient(fshttp.NewClient(ctx))
 
 	fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
 	if config.Confirm(false) {
 		deviceRegistration, err := registerDevice(ctx, srv)
 		if err != nil {
-			log.Fatalf("Failed to register device: %v", err)
+			return errors.Wrap(err, "failed to register device")
 		}
 
 		m.Set(configClientID, deviceRegistration.ClientID)
@@ -315,18 +319,18 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) {
 
 	token, err := doAuthV1(ctx, srv, username, password)
 	if err != nil {
-		log.Fatalf("Failed to get oauth token: %s", err)
+		return errors.Wrap(err, "failed to get oauth token")
 	}
 	err = oauthutil.PutToken(name, m, &token, true)
 	if err != nil {
-		log.Fatalf("Error while saving token: %s", err)
+		return errors.Wrap(err, "error while saving token")
 	}
 
 	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
 	if config.Confirm(false) {
 		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 		if err != nil {
-			log.Fatalf("Failed to load oAuthClient: %s", err)
+			return errors.Wrap(err, "failed to load oAuthClient")
 		}
 
 		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
@@ -334,13 +338,14 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) {
 
 		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
 		if err != nil {
-			log.Fatalf("Failed to setup mountpoint: %s", err)
+			return errors.Wrap(err, "failed to setup mountpoint")
 		}
 		m.Set(configDevice, device)
 		m.Set(configMountpoint, mountpoint)
 	}
 
 	m.Set("configVersion", strconv.Itoa(v1configVersion))
+	return nil
 }
 
 // registerDevice register a new device for use with the jottacloud API
@@ -415,7 +420,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
 }
 
 // v2config configure a jottacloud backend using the modern JottaCli token based authentication
-func v2config(ctx context.Context, name string, m configmap.Mapper) {
+func v2config(ctx context.Context, name string, m configmap.Mapper) error {
 	srv := rest.NewClient(fshttp.NewClient(ctx))
 
 	fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
@@ -427,31 +432,32 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
 
 	token, err := doAuthV2(ctx, srv, loginToken, m)
 	if err != nil {
-		log.Fatalf("Failed to get oauth token: %s", err)
+		return errors.Wrap(err, "failed to get oauth token")
 	}
 	err = oauthutil.PutToken(name, m, &token, true)
 	if err != nil {
-		log.Fatalf("Error while saving token: %s", err)
+		return errors.Wrap(err, "error while saving token")
 	}
 
 	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
 	if config.Confirm(false) {
 		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 		if err != nil {
-			log.Fatalf("Failed to load oAuthClient: %s", err)
+			return errors.Wrap(err, "failed to load oAuthClient")
 		}
 
 		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
 		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
 		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
 		if err != nil {
-			log.Fatalf("Failed to setup mountpoint: %s", err)
+			return errors.Wrap(err, "failed to setup mountpoint")
 		}
 		m.Set(configDevice, device)
 		m.Set(configMountpoint, mountpoint)
 	}
 
 	m.Set("configVersion", strconv.Itoa(configVersion))
+	return nil
 }
 
 // doAuthV2 runs the actual token request for V2 authentication
@@ -615,7 +621,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -854,7 +860,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -883,7 +889,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var result api.JottaFolder
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -995,7 +1001,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1101,7 +1107,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't purge directory")
@@ -1140,7 +1146,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1268,7 +1274,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var result api.JottaFile
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -1446,7 +1452,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1559,7 +1565,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var response api.AllocateFileResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1624,7 +1630,7 @@ func (o *Object) Remove(ctx context.Context) error {
 
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 

backend/local/local.go

@@ -42,8 +42,9 @@ func init() {
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
-			Name: "nounc",
-			Help: "Disable UNC (long path names) conversion on Windows",
+			Name:     "nounc",
+			Help:     "Disable UNC (long path names) conversion on Windows",
+			Advanced: runtime.GOOS != "windows",
 			Examples: []fs.OptionExample{{
 				Value: "true",
 				Help:  "Disables long file names",
@@ -1144,6 +1145,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			err = file.PreAllocate(src.Size(), f)
 			if err != nil {
 				fs.Debugf(o, "Failed to pre-allocate: %v", err)
+				if err == file.ErrDiskFull {
+					_ = f.Close()
+					return err
+				}
 			}
 		}
 		out = f

backend/mailru/api/helpers.go

@@ -6,8 +6,8 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"io"
-	"log"
 	"time"
 
 	"github.com/pkg/errors"
@@ -48,7 +48,7 @@ func (w *BinWriter) Reader() io.Reader {
 // WritePu16 writes a short as unsigned varint
 func (w *BinWriter) WritePu16(val int) {
 	if val < 0 || val > 65535 {
-		log.Fatalf("Invalid UInt16 %v", val)
+		panic(fmt.Sprintf("Invalid UInt16 %v", val))
 	}
 	w.WritePu64(int64(val))
 }
@@ -56,7 +56,7 @@ func (w *BinWriter) WritePu16(val int) {
 // WritePu32 writes a signed long as unsigned varint
 func (w *BinWriter) WritePu32(val int64) {
 	if val < 0 || val > 4294967295 {
-		log.Fatalf("Invalid UInt32 %v", val)
+		panic(fmt.Sprintf("Invalid UInt32 %v", val))
 	}
 	w.WritePu64(val)
 }
@@ -64,7 +64,7 @@ func (w *BinWriter) WritePu32(val int64) {
 // WritePu64 writes an unsigned (actually, signed) long as unsigned varint
 func (w *BinWriter) WritePu64(val int64) {
 	if val < 0 {
-		log.Fatalf("Invalid UInt64 %v", val)
+		panic(fmt.Sprintf("Invalid UInt64 %v", val))
 	}
 	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
 }
@@ -123,7 +123,7 @@ func (r *BinReader) check(err error) bool {
 		r.err = err
 	}
 	if err != io.EOF {
-		log.Fatalf("Error parsing response: %v", err)
+		panic(fmt.Sprintf("Error parsing response: %v", err))
 	}
 	return false
 }

backend/mailru/mailru.go

@@ -234,7 +234,10 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this response and err
 // deserve to be retried. It returns the err as a convenience.
 // Retries password authorization (once) in a special case of access denied.
-func shouldRetry(res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
+func shouldRetry(ctx context.Context, res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if res != nil && res.StatusCode == 403 && f.opt.Password != "" && !f.passFailed {
 		reAuthErr := f.reAuthorize(opts, err)
 		return reAuthErr == nil, err // return an original error
@@ -600,7 +603,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 	var info api.ItemInfoResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -736,7 +739,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int)
 	)
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -800,7 +803,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1073,7 +1076,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1216,7 +1219,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
 	var response api.GenericResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 
 	switch {
@@ -1288,7 +1291,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var response api.GenericBodyResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -1392,7 +1395,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1483,7 +1486,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var response api.GenericBodyResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 
 	if err == nil && response.Body != "" {
@@ -1524,7 +1527,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	var response api.CleanupResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 	if err != nil {
 		return err
@@ -1557,7 +1560,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var info api.UserInfoResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(res, err, f, &opts)
+		return shouldRetry(ctx, res, err, f, &opts)
 	})
 	if err != nil {
 		return nil, err
@@ -2076,7 +2079,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	var res *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(res, err, o.fs, &opts)
+		return shouldRetry(ctx, res, err, o.fs, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -2172,7 +2175,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		opts.RootURL = server
 		res, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(res, err, o.fs, &opts)
+		return shouldRetry(ctx, res, err, o.fs, &opts)
 	})
 	if err != nil {
 		if res != nil && res.Body != nil {

backend/mega/mega.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -158,7 +159,10 @@ func parsePath(path string) (root string) {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// Let the mega library handle the low level retries
 	return false, err
 	/*
@@ -171,8 +175,8 @@ func shouldRetry(err error) (bool, error) {
 }
 
 // readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) {
-	rootNode, err := f.findRoot(false)
+func (f *Fs) readMetaDataForPath(ctx context.Context, remote string) (info *mega.Node, err error) {
+	rootNode, err := f.findRoot(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -237,7 +241,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 
 	// Find the root node and check if it is a file or not
-	_, err = f.findRoot(false)
+	_, err = f.findRoot(ctx, false)
 	switch err {
 	case nil:
 		// root node found and is a directory
@@ -307,8 +311,8 @@ func (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err
 // lookupDir looks up the node for the directory of the name given
 //
 // if create is true it tries to create the root directory if not found
-func (f *Fs) lookupDir(dir string) (*mega.Node, error) {
-	rootNode, err := f.findRoot(false)
+func (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {
+	rootNode, err := f.findRoot(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -316,15 +320,15 @@ func (f *Fs) lookupDir(dir string) (*mega.Node, error) {
 }
 
 // lookupParentDir finds the parent node for the remote passed in
-func (f *Fs) lookupParentDir(remote string) (dirNode *mega.Node, leaf string, err error) {
+func (f *Fs) lookupParentDir(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
 	parent, leaf := path.Split(remote)
-	dirNode, err = f.lookupDir(parent)
+	dirNode, err = f.lookupDir(ctx, parent)
 	return dirNode, leaf, err
 }
 
 // mkdir makes the directory and any parent directories for the
 // directory of the name given
-func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error) {
+func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *mega.Node, err error) {
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
@@ -358,7 +362,7 @@ func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error)
 		// create directory called name in node
 		err = f.pacer.Call(func() (bool, error) {
 			node, err = f.srv.CreateDir(name, node)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "mkdir create node failed")
@@ -368,20 +372,20 @@ func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error)
 }
 
 // mkdirParent creates the parent directory of remote
-func (f *Fs) mkdirParent(remote string) (dirNode *mega.Node, leaf string, err error) {
-	rootNode, err := f.findRoot(true)
+func (f *Fs) mkdirParent(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
+	rootNode, err := f.findRoot(ctx, true)
 	if err != nil {
 		return nil, "", err
 	}
 	parent, leaf := path.Split(remote)
-	dirNode, err = f.mkdir(rootNode, parent)
+	dirNode, err = f.mkdir(ctx, rootNode, parent)
 	return dirNode, leaf, err
 }
 
 // findRoot looks up the root directory node and returns it.
 //
 // if create is true it tries to create the root directory if not found
-func (f *Fs) findRoot(create bool) (*mega.Node, error) {
+func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
 	f.rootNodeMu.Lock()
 	defer f.rootNodeMu.Unlock()
 
@@ -403,7 +407,7 @@ func (f *Fs) findRoot(create bool) (*mega.Node, error) {
 	}
 
 	//..not found so create the root directory
-	f._rootNode, err = f.mkdir(absRoot, f.root)
+	f._rootNode, err = f.mkdir(ctx, absRoot, f.root)
 	return f._rootNode, err
 }
 
@@ -433,7 +437,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))
 		deleteErr := f.pacer.Call(func() (bool, error) {
 			err := f.srv.Delete(item, true)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if deleteErr != nil {
 			err = deleteErr
@@ -447,7 +451,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.Node) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -457,7 +461,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData() // reads info and meta, returning an error
+		err = o.readMetaData(ctx) // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -468,7 +472,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
 // list the objects into the function supplied
@@ -506,7 +510,7 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	dirNode, err := f.lookupDir(dir)
+	dirNode, err := f.lookupDir(ctx, dir)
 	if err != nil {
 		return nil, err
 	}
@@ -518,7 +522,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash())
 			entries = append(entries, d)
 		case mega.FILE:
-			o, err := f.newObjectWithInfo(remote, info)
+			o, err := f.newObjectWithInfo(ctx, remote, info)
 			if err != nil {
 				iErr = err
 				return true
@@ -542,8 +546,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Returns the dirNode, object, leaf and error
 //
 // Used to create new objects
-func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
-	dirNode, leaf, err = f.mkdirParent(remote)
+func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
+	dirNode, leaf, err = f.mkdirParent(ctx, remote)
 	if err != nil {
 		return nil, nil, leaf, err
 	}
@@ -565,7 +569,7 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
 	switch err {
 	case nil:
 		return existingObj, existingObj.Update(ctx, in, src, options...)
@@ -591,7 +595,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	size := src.Size()
 	modTime := src.ModTime(ctx)
 
-	o, _, _, err := f.createObject(remote, modTime, size)
+	o, _, _, err := f.createObject(ctx, remote, modTime, size)
 	if err != nil {
 		return nil, err
 	}
@@ -600,30 +604,30 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the directory if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	rootNode, err := f.findRoot(true)
+	rootNode, err := f.findRoot(ctx, true)
 	if err != nil {
 		return err
 	}
-	_, err = f.mkdir(rootNode, dir)
+	_, err = f.mkdir(ctx, rootNode, dir)
 	return errors.Wrap(err, "Mkdir failed")
 }
 
 // deleteNode removes a file or directory, observing useTrash
-func (f *Fs) deleteNode(node *mega.Node) (err error) {
+func (f *Fs) deleteNode(ctx context.Context, node *mega.Node) (err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		err = f.srv.Delete(node, f.opt.HardDelete)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return err
 }
 
 // purgeCheck removes the directory dir, if check is set then it
 // refuses to do so if it has anything in
-func (f *Fs) purgeCheck(dir string, check bool) error {
+func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
-	rootNode, err := f.findRoot(false)
+	rootNode, err := f.findRoot(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -644,7 +648,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 
 	waitEvent := f.srv.WaitEventsStart()
 
-	err = f.deleteNode(dirNode)
+	err = f.deleteNode(ctx, dirNode)
 	if err != nil {
 		return errors.Wrap(err, "delete directory node failed")
 	}
@@ -662,7 +666,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 //
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(dir, true)
+	return f.purgeCheck(ctx, dir, true)
 }
 
 // Precision return the precision of this Fs
@@ -676,13 +680,13 @@ func (f *Fs) Precision() time.Duration {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(dir, false)
+	return f.purgeCheck(ctx, dir, false)
 }
 
 // move a file or folder (srcFs, srcRemote, info) to (f, dstRemote)
 //
 // info will be updates
-func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
+func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
 	var (
 		dstFs                  = f
 		srcDirNode, dstDirNode *mega.Node
@@ -692,12 +696,12 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 
 	if dstRemote != "" {
 		// lookup or create the destination parent directory
-		dstDirNode, dstLeaf, err = dstFs.mkdirParent(dstRemote)
+		dstDirNode, dstLeaf, err = dstFs.mkdirParent(ctx, dstRemote)
 	} else {
 		// find or create the parent of the root directory
 		absRoot := dstFs.srv.FS.GetRoot()
 		dstParent, dstLeaf = path.Split(dstFs.root)
-		dstDirNode, err = dstFs.mkdir(absRoot, dstParent)
+		dstDirNode, err = dstFs.mkdir(ctx, absRoot, dstParent)
 	}
 	if err != nil {
 		return errors.Wrap(err, "server-side move failed to make dst parent dir")
@@ -705,7 +709,7 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 
 	if srcRemote != "" {
 		// lookup the existing parent directory
-		srcDirNode, srcLeaf, err = srcFs.lookupParentDir(srcRemote)
+		srcDirNode, srcLeaf, err = srcFs.lookupParentDir(ctx, srcRemote)
 	} else {
 		// lookup the existing root parent
 		absRoot := srcFs.srv.FS.GetRoot()
@@ -721,7 +725,7 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 		//log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName())
 		err = f.pacer.Call(func() (bool, error) {
 			err = f.srv.Move(info, dstDirNode)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "server-side move failed")
@@ -735,7 +739,7 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 		//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
 		err = f.pacer.Call(func() (bool, error) {
 			err = f.srv.Rename(info, f.opt.Enc.FromStandardName(dstLeaf))
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "server-side rename failed")
@@ -767,7 +771,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Do the move
-	err := f.move(remote, srcObj.fs, srcObj.remote, srcObj.info)
+	err := f.move(ctx, remote, srcObj.fs, srcObj.remote, srcObj.info)
 	if err != nil {
 		return nil, err
 	}
@@ -798,13 +802,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	// find the source
-	info, err := srcFs.lookupDir(srcRemote)
+	info, err := srcFs.lookupDir(ctx, srcRemote)
 	if err != nil {
 		return err
 	}
 
 	// check the destination doesn't exist
-	_, err = dstFs.lookupDir(dstRemote)
+	_, err = dstFs.lookupDir(ctx, dstRemote)
 	if err == nil {
 		return fs.ErrorDirExists
 	} else if err != fs.ErrorDirNotFound {
@@ -812,7 +816,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	// Do the move
-	err = f.move(dstRemote, srcFs, srcRemote, info)
+	err = f.move(ctx, dstRemote, srcFs, srcRemote, info)
 	if err != nil {
 		return err
 	}
@@ -838,7 +842,7 @@ func (f *Fs) Hashes() hash.Set {
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
-	root, err := f.findRoot(false)
+	root, err := f.findRoot(ctx, false)
 	if err != nil {
 		return "", errors.Wrap(err, "PublicLink failed to find root node")
 	}
@@ -886,7 +890,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 			fs.Infof(srcDir, "merging %q", f.opt.Enc.ToStandardName(info.GetName()))
 			err = f.pacer.Call(func() (bool, error) {
 				err = f.srv.Move(info, dstDirNode)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err != nil {
 				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", f.opt.Enc.ToStandardName(info.GetName()), srcDir)
@@ -894,7 +898,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
-		err = f.deleteNode(srcDirNode)
+		err = f.deleteNode(ctx, srcDirNode)
 		if err != nil {
 			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
@@ -908,7 +912,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		q, err = f.srv.GetQuota()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get Mega Quota")
@@ -963,11 +967,11 @@ func (o *Object) setMetaData(info *mega.Node) (err error) {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // it also sets the info
-func (o *Object) readMetaData() (err error) {
+func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.info != nil {
 		return nil
 	}
-	info, err := o.fs.readMetaDataForPath(o.remote)
+	info, err := o.fs.readMetaDataForPath(ctx, o.remote)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			err = fs.ErrorObjectNotFound
@@ -998,6 +1002,7 @@ func (o *Object) Storable() bool {
 
 // openObject represents a download in progress
 type openObject struct {
+	ctx    context.Context
 	mu     sync.Mutex
 	o      *Object
 	d      *mega.Download
@@ -1008,14 +1013,14 @@ type openObject struct {
 }
 
 // get the next chunk
-func (oo *openObject) getChunk() (err error) {
+func (oo *openObject) getChunk(ctx context.Context) (err error) {
 	if oo.id >= oo.d.Chunks() {
 		return io.EOF
 	}
 	var chunk []byte
 	err = oo.o.fs.pacer.Call(func() (bool, error) {
 		chunk, err = oo.d.DownloadChunk(oo.id)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
@@ -1045,7 +1050,7 @@ func (oo *openObject) Read(p []byte) (n int, err error) {
 		oo.skip -= int64(size)
 	}
 	if len(oo.chunk) == 0 {
-		err = oo.getChunk()
+		err = oo.getChunk(oo.ctx)
 		if err != nil {
 			return 0, err
 		}
@@ -1068,7 +1073,7 @@ func (oo *openObject) Close() (err error) {
 	}
 	err = oo.o.fs.pacer.Call(func() (bool, error) {
 		err = oo.d.Finish()
-		return shouldRetry(err)
+		return shouldRetry(oo.ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to finish download")
@@ -1096,13 +1101,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var d *mega.Download
 	err = o.fs.pacer.Call(func() (bool, error) {
 		d, err = o.fs.srv.NewDownload(o.info)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open download file failed")
 	}
 
 	oo := &openObject{
+		ctx:  ctx,
 		o:    o,
 		d:    d,
 		skip: offset,
@@ -1125,7 +1131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the parent directory
-	dirNode, leaf, err := o.fs.mkdirParent(remote)
+	dirNode, leaf, err := o.fs.mkdirParent(ctx, remote)
 	if err != nil {
 		return errors.Wrap(err, "update make parent dir failed")
 	}
@@ -1133,7 +1139,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var u *mega.Upload
 	err = o.fs.pacer.Call(func() (bool, error) {
 		u, err = o.fs.srv.NewUpload(dirNode, o.fs.opt.Enc.FromStandardName(leaf), size)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file failed to create session")
@@ -1154,7 +1160,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		err = o.fs.pacer.Call(func() (bool, error) {
 			err = u.UploadChunk(id, chunk)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "upload file failed to upload chunk")
@@ -1165,7 +1171,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var info *mega.Node
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, err = u.Finish()
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to finish upload")
@@ -1173,7 +1179,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// If the upload succeeded and the original object existed, then delete it
 	if o.info != nil {
-		err = o.fs.deleteNode(o.info)
+		err = o.fs.deleteNode(ctx, o.info)
 		if err != nil {
 			return errors.Wrap(err, "upload failed to remove old version")
 		}
@@ -1185,7 +1191,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	err := o.fs.deleteNode(o.info)
+	err := o.fs.deleteNode(ctx, o.info)
 	if err != nil {
 		return errors.Wrap(err, "Remove object failed")
 	}

backend/onedrive/onedrive.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -99,7 +98,7 @@ func init() {
 		Name:        "onedrive",
 		Description: "Microsoft OneDrive",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			region, _ := m.Get("region")
 			graphURL := graphAPIEndpoint[region] + "/v1.0"
 			oauthConfig.Endpoint = oauth2.Endpoint{
@@ -109,13 +108,12 @@ func init() {
 			ci := fs.GetConfig(ctx)
 			err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
-				return
+				return errors.Wrap(err, "failed to configure token")
 			}
 
 			// Stop if we are running non-interactive config
 			if ci.AutoConfirm {
-				return
+				return nil
 			}
 
 			type driveResource struct {
@@ -138,7 +136,7 @@ func init() {
 
 			oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 			if err != nil {
-				log.Fatalf("Failed to configure OneDrive: %v", err)
+				return errors.Wrap(err, "failed to configure OneDrive")
 			}
 			srv := rest.NewClient(oAuthClient)
 
@@ -203,18 +201,17 @@ func init() {
 				sites := siteResponse{}
 				_, err := srv.CallJSON(ctx, &opts, nil, &sites)
 				if err != nil {
-					log.Fatalf("Failed to query available sites: %v", err)
+					return errors.Wrap(err, "failed to query available sites")
 				}
 
 				if len(sites.Sites) == 0 {
-					log.Fatalf("Search for '%s' returned no results", searchTerm)
-				} else {
-					fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
-					for index, site := range sites.Sites {
-						fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
-					}
-					siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
+					return errors.Errorf("search for %q returned no results", searchTerm)
 				}
+				fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
+				for index, site := range sites.Sites {
+					fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
+				}
+				siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
 			}
 
 			// if we use server-relative URL for finding the drive
@@ -227,7 +224,7 @@ func init() {
 				site := siteResource{}
 				_, err := srv.CallJSON(ctx, &opts, nil, &site)
 				if err != nil {
-					log.Fatalf("Failed to query available site by relative path: %v", err)
+					return errors.Wrap(err, "failed to query available site by relative path")
 				}
 				siteID = site.SiteID
 			}
@@ -247,7 +244,7 @@ func init() {
 				drives := drivesResponse{}
 				_, err := srv.CallJSON(ctx, &opts, nil, &drives)
 				if err != nil {
-					log.Fatalf("Failed to query available drives: %v", err)
+					return errors.Wrap(err, "failed to query available drives")
 				}
 
 				// Also call /me/drive as sometimes /me/drives doesn't return it #4068
@@ -256,7 +253,7 @@ func init() {
 					meDrive := driveResource{}
 					_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
 					if err != nil {
-						log.Fatalf("Failed to query available drives: %v", err)
+						return errors.Wrap(err, "failed to query available drives")
 					}
 					found := false
 					for _, drive := range drives.Drives {
@@ -273,14 +270,13 @@ func init() {
 				}
 
 				if len(drives.Drives) == 0 {
-					log.Fatalf("No drives found")
-				} else {
-					fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
-					for index, drive := range drives.Drives {
-						fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
-					}
-					finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
+					return errors.New("no drives found")
 				}
+				fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
+				for index, drive := range drives.Drives {
+					fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
+				}
+				finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
 			}
 
 			// Test the driveID and get drive type
@@ -291,18 +287,18 @@ func init() {
 			var rootItem api.Item
 			_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
 			if err != nil {
-				log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
+				return errors.Wrapf(err, "failed to query root for drive %s", finalDriveID)
 			}
 
 			fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
 			// This does not work, YET :)
 			if !config.ConfirmWithConfig(ctx, m, "config_drive_ok", true) {
-				log.Fatalf("Cancelled by user")
+				return errors.New("cancelled by user")
 			}
 
 			m.Set(configDriveID, finalDriveID)
 			m.Set(configDriveType, rootItem.ParentReference.DriveType)
-			config.SaveConfig()
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "region",
@@ -362,6 +358,11 @@ This will only work if you are copying between two OneDrive *Personal* drives AN
 the files to copy are already shared between them.  In other cases, rclone will
 fall back to normal copy (which will be slightly slower).`,
 			Advanced: true,
+		}, {
+			Name:     "list_chunk",
+			Help:     "Size of listing chunk.",
+			Default:  1000,
+			Advanced: true,
 		}, {
 			Name:    "no_versions",
 			Default: false,
@@ -469,6 +470,7 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
+	ListChunk               int64                `config:"list_chunk"`
 	NoVersions              bool                 `config:"no_versions"`
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
@@ -550,7 +552,10 @@ var errAsyncJobAccessDenied = errors.New("async job failed - access denied")
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	retry := false
 	if resp != nil {
 		switch resp.StatusCode {
@@ -558,6 +563,9 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 			if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
 				retry = true
 				fs.Debugf(nil, "Should retry: %v", err)
+			} else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") {
+				retry = true
+				fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
 			}
 		case 429: // Too Many Requests.
 			// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
@@ -597,7 +605,7 @@ func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID s
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	return info, resp, err
@@ -613,7 +621,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		opts.Path = strings.TrimSuffix(opts.Path, ":")
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		return info, resp, err
 	}
@@ -869,7 +877,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -894,14 +902,14 @@ type listAllFn func(*api.Item) bool
 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	// Top parameter asks for bigger pages of data
 	// https://dev.onedrive.com/odata/optional-query-parameters.htm
-	opts := f.newOptsCall(dirID, "GET", "/children?$top=1000")
+	opts := f.newOptsCall(dirID, "GET", fmt.Sprintf("/children?$top=%d", f.opt.ListChunk))
 OUTER:
 	for {
 		var result api.ListChildrenResponse
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -1038,7 +1046,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1194,7 +1202,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyReq, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1287,7 +1295,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1354,7 +1362,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1380,7 +1388,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -1421,7 +1429,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		Password: f.opt.LinkPassword,
 	}
 
-	if expire < fs.Duration(time.Hour*24*365*100) {
+	if expire < fs.DurationOff {
 		expiry := time.Now().Add(time.Duration(expire))
 		share.Expiry = &expiry
 	}
@@ -1430,7 +1438,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var result api.CreateShareLinkResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &share, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -1475,7 +1483,7 @@ func (o *Object) deleteVersions(ctx context.Context) error {
 	var versions api.VersionsResponse
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1502,7 +1510,7 @@ func (o *Object) deleteVersion(ctx context.Context, ID string) error {
 	opts.NoResponse = true
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1653,7 +1661,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	// Remove versions if required
 	if o.fs.opt.NoVersions {
@@ -1695,7 +1703,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1723,7 +1731,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 				err = errors.New(err.Error() + " (is it a OneNote file?)")
 			}
 		}
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return response, err
 }
@@ -1738,7 +1746,7 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return 0, err
@@ -1798,11 +1806,11 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			return true, errors.Wrapf(err, "retry this chunk skipping %d bytes", skip)
 		}
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		body, err = rest.ReadBody(resp)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		if resp.StatusCode == 200 || resp.StatusCode == 201 {
 			// we are done :)
@@ -1825,7 +1833,7 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return
 }
@@ -1849,7 +1857,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		fs.Debugf(o, "Cancelling multipart upload: %v", err)
 		cancelErr := o.cancelUploadSession(ctx, uploadURL)
 		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+			fs.Logf(o, "Failed to cancel multipart upload: %v (upload failed due to: %v)", cancelErr, err)
 		}
 	})()
 
@@ -1896,7 +1904,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 				err = errors.New(err.Error() + " (is it a OneNote file?)")
 			}
 		}
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err

backend/opendrive/opendrive.go

@@ -119,6 +119,7 @@ type Object struct {
 	fs      *Fs       // what this object is part of
 	remote  string    // The remote path
 	id      string    // ID of the file
+	parent  string    // ID of the parent directory
 	modTime time.Time // The modified time of the object if known
 	md5     string    // MD5 hash if known
 	size    int64     // Size of the object
@@ -206,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			Path:   "/session/login.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create session")
@@ -233,7 +234,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(ctx, remote, nil)
+		_, err := tempF.newObjectWithInfo(ctx, remote, nil, "")
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -293,7 +294,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 			Path:       "/folder/remove.json",
 		}
 		resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -388,7 +389,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -445,7 +446,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -494,7 +495,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			Path:   "/folder/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error %v", err)
@@ -517,7 +518,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, parent string) (fs.Object, error) {
 	// fs.Debugf(nil, "newObjectWithInfo(%s, %v)", remote, file)
 
 	var o *Object
@@ -526,6 +527,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (
 			fs:      f,
 			remote:  remote,
 			id:      file.FileID,
+			parent:  parent,
 			modTime: time.Unix(file.DateModified, 0),
 			size:    file.Size,
 			md5:     file.FileHash,
@@ -548,7 +550,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	// fs.Debugf(nil, "NewObject(\"%s\")", remote)
-	return f.newObjectWithInfo(ctx, remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil, "")
 }
 
 // Creates from the parameters passed in a half finished Object which
@@ -581,7 +583,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -631,7 +633,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 				Path:    "/upload/create_file.json",
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
-			return o.fs.shouldRetry(resp, err)
+			return o.fs.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create file")
@@ -657,7 +659,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -683,7 +688,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			Path:   "/folder.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -711,7 +716,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 			Path:   "/folder/list.json/" + f.session.SessionID + "/" + pathID,
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", false, errors.Wrap(err, "failed to get folder list")
@@ -754,7 +759,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	folderList := FolderList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get folder list")
@@ -768,6 +773,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		f.dirCache.Put(remote, folder.FolderID)
 		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
+		d.SetParentID(directoryID)
 		entries = append(entries, d)
 	}
 
@@ -775,7 +781,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		file.Name = f.opt.Enc.ToStandardName(file.Name)
 		// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
 		remote := path.Join(dir, file.Name)
-		o, err := f.newObjectWithInfo(ctx, remote, &file)
+		o, err := f.newObjectWithInfo(ctx, remote, &file, directoryID)
 		if err != nil {
 			return nil, err
 		}
@@ -842,7 +848,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 
 	o.modTime = modTime
@@ -862,7 +868,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open file)")
@@ -881,7 +887,7 @@ func (o *Object) Remove(ctx context.Context) error {
 			Path:       "/file.json/" + o.fs.session.SessionID + "/" + o.id,
 		}
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -910,7 +916,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:    "/upload/open_file_upload.json",
 		}
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -954,7 +960,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
-			return o.fs.shouldRetry(resp, err)
+			return o.fs.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to create file")
@@ -977,7 +983,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:   "/upload/close_file_upload.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -1003,7 +1009,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:       "/file/access.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1029,7 +1035,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 				o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get folder list")
@@ -1053,6 +1059,11 @@ func (o *Object) ID() string {
 	return o.id
 }
 
+// ParentID returns the ID of the Object parent directory if known, or "" if not
+func (o *Object) ParentID() string {
+	return o.parent
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1063,4 +1074,5 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
+	_ fs.ParentIDer      = (*Object)(nil)
 )

backend/pcloud/pcloud.go

@@ -12,7 +12,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -72,7 +71,7 @@ func init() {
 		Name:        "pcloud",
 		Description: "Pcloud",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			optc := new(Options)
 			err := configstruct.Set(m, optc)
 			if err != nil {
@@ -100,8 +99,9 @@ func init() {
 			}
 			err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     config.ConfigEncoding,
@@ -213,7 +213,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	doRetry := false
 
 	// Check if it is an api.Error
@@ -405,7 +408,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -460,7 +463,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -597,7 +600,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -662,7 +665,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -700,7 +703,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -740,7 +743,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -787,7 +790,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -814,7 +817,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -838,7 +841,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -869,7 +872,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
 		err = q.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -927,7 +930,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -1046,7 +1049,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1072,7 +1075,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1134,7 +1137,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// opts.Body=0), so upload it as a multipart form POST with
 	// Content-Length set.
 	if size == 0 {
-		formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
+		formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, opts.Parameters, "content", leaf)
 		if err != nil {
 			return errors.Wrap(err, "failed to make multipart upload for 0 length file")
 		}
@@ -1151,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// sometimes pcloud leaves a half complete file on
@@ -1181,7 +1184,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 

backend/premiumizeme/premiumizeme.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
@@ -78,11 +77,12 @@ func init() {
 		Name:        "premiumizeme",
 		Description: "premiumize.me",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			err := oauthutil.Config(ctx, "premiumizeme", name, m, oauthConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: []fs.Option{{
 			Name: "api_key",
@@ -176,7 +176,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -370,7 +373,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -407,7 +410,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -581,7 +584,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -660,7 +663,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
 		var result api.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Move http")
@@ -769,7 +772,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "CreateDir http")
@@ -896,7 +899,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -934,7 +937,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		// Just check the download URL resolves - sometimes
 		// the URLs returned by premiumize.me don't resolve so
@@ -993,7 +996,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.Response
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file http")
@@ -1035,7 +1038,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rename http")
@@ -1060,7 +1063,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "remove http")

backend/putio/error.go

@@ -1,6 +1,7 @@
 package putio
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
@@ -29,7 +30,10 @@ func (e *statusCodeError) Temporary() bool {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	if err == nil {
 		return false, nil
 	}

backend/putio/fs.go

@@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
 		entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	return itoa(entry.ID), err
 }
@@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing file: %d", fileID)
 		children, _, err = f.client.Files.List(ctx, fileID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
@@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing files inside List: %d", parentID)
 		children, _, err = f.client.Files.List(ctx, parentID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting file: %d", fileID)
 		entry, err = f.client.Files.Get(ctx, fileID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -295,7 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 		req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
 		fs.OpenOptionAddHTTPHeaders(req.Header, options)
 		resp, err := f.oAuthClient.Do(req)
-		retry, err := shouldRetry(err)
+		retry, err := shouldRetry(ctx, err)
 		if retry {
 			return true, err
 		}
@@ -320,7 +320,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 		err = f.pacer.Call(func() (bool, error) {
 			fs.Debugf(f, "Sending zero length chunk")
 			_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		return
 	}
@@ -344,13 +344,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				// Get file offset and seek to the position
 				offset, err := f.getServerOffset(ctx, location)
 				if err != nil {
-					return shouldRetry(err)
+					return shouldRetry(ctx, err)
 				}
 				sentBytes := offset - chunkStart
 				fs.Debugf(f, "sentBytes: %d", sentBytes)
 				_, err = chunk.Seek(sentBytes, io.SeekStart)
 				if err != nil {
-					return shouldRetry(err)
+					return shouldRetry(ctx, err)
 				}
 				transferOffset = offset
 				reqSize = chunkSize - sentBytes
@@ -367,7 +367,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				offsetMismatch = true
 				return true, errors.New("connection broken")
 			}
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return
@@ -479,7 +479,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		err = f.pacer.Call(func() (bool, error) {
 			// fs.Debugf(f, "listing files: %d", dirID)
 			children, _, err = f.client.Files.List(ctx, dirID)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
@@ -493,7 +493,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "deleting file: %d", dirID)
 		err = f.client.Files.Delete(ctx, dirID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	f.dirCache.FlushDir(dir)
 	return err
@@ -552,7 +552,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -591,7 +591,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -631,7 +631,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	srcFs.dirCache.FlushDir(srcRemote)
 	return err
@@ -644,7 +644,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting account info")
 		ai, err = f.client.Account.Info(ctx)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -678,6 +678,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		}
 		// fs.Debugf(f, "emptying trash")
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 }

backend/putio/object.go

@@ -145,7 +145,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
 			return false, fs.ErrorObjectNotFound
 		}
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -220,7 +220,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var storageURL string
 	err = o.fs.pacer.Call(func() (bool, error) {
 		storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return
@@ -231,7 +231,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	err = o.fs.pacer.Call(func() (bool, error) {
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil)
 		if err != nil {
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		}
 		req.Header.Set("User-Agent", o.fs.client.UserAgent)
 
@@ -241,7 +241,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		// fs.Debugf(o, "opening file: id=%d", o.file.ID)
 		resp, err = o.fs.httpClient.Do(req)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
 		_ = resp.Body.Close()
@@ -283,6 +283,6 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	return o.fs.pacer.Call(func() (bool, error) {
 		// fs.Debugf(o, "removing file: id=%d", o.file.ID)
 		err = o.fs.client.Files.Delete(ctx, o.file.ID)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 }

backend/putio/putio.go

@@ -2,10 +2,10 @@ package putio
 
 import (
 	"context"
-	"log"
 	"regexp"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -60,14 +60,15 @@ func init() {
 		Name:        "putio",
 		Description: "Put.io",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			opt := oauthutil.Options{
 				NoOffline: true,
 			}
 			err := oauthutil.Config(ctx, "putio", name, m, putioConfig, &opt)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: []fs.Option{{
 			Name:     config.ConfigEncoding,

backend/s3/s3.go

@@ -26,7 +26,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
@@ -1399,7 +1398,10 @@ var retryErrorCodes = []int{
 //S3 is pretty resilient, and the built in retry handling is probably sufficient
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
 		// Simple case, check the original embedded error in case it's generically retryable
@@ -1411,7 +1413,7 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
 			// 301 if wrong region for bucket - can only update if running from a bucket
 			if f.rootBucket != "" {
 				if reqErr.StatusCode() == http.StatusMovedPermanently {
-					urfbErr := f.updateRegionForBucket(f.rootBucket)
+					urfbErr := f.updateRegionForBucket(ctx, f.rootBucket)
 					if urfbErr != nil {
 						fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
 						return false, err
@@ -1508,11 +1510,6 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 			}),
 			ExpiryWindow: 3 * time.Minute,
 		},
-
-		// Pick up IAM role if we are in EKS
-		&stscreds.WebIdentityRoleProvider{
-			ExpiryWindow: 3 * time.Minute,
-		},
 	}
 	cred := credentials.NewChainCredentials(providers)
 
@@ -1559,6 +1556,8 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
 		// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
 		awsSessionOpts.SharedConfigState = session.SharedConfigEnable
+		// Set the name of the profile if supplied
+		awsSessionOpts.Profile = opt.Profile
 	}
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
@@ -1741,7 +1740,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 }
 
 // Gets the bucket location
-func (f *Fs) getBucketLocation(bucket string) (string, error) {
+func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, error) {
 	req := s3.GetBucketLocationInput{
 		Bucket: &bucket,
 	}
@@ -1749,7 +1748,7 @@ func (f *Fs) getBucketLocation(bucket string) (string, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return "", err
@@ -1759,8 +1758,8 @@ func (f *Fs) getBucketLocation(bucket string) (string, error) {
 
 // Updates the region for the bucket by reading the region from the
 // bucket then updating the session.
-func (f *Fs) updateRegionForBucket(bucket string) error {
-	region, err := f.getBucketLocation(bucket)
+func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
+	region, err := f.getBucketLocation(ctx, bucket)
 	if err != nil {
 		return errors.Wrap(err, "reading bucket location failed")
 	}
@@ -1854,7 +1853,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					}
 				}
 			}
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -2001,7 +2000,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 	var resp *s3.ListBucketsOutput
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.ListBucketsWithContext(ctx, &req)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
@@ -2116,7 +2115,7 @@ func (f *Fs) bucketExists(ctx context.Context, bucket string) (bool, error) {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.HeadBucketWithContext(ctx, &req)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err == nil {
 		return true, nil
@@ -2152,7 +2151,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 		}
 		err := f.pacer.Call(func() (bool, error) {
 			_, err := f.c.CreateBucketWithContext(ctx, &req)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err == nil {
 			fs.Infof(f, "Bucket %q created with ACL %q", bucket, f.opt.BucketACL)
@@ -2182,7 +2181,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		}
 		err := f.pacer.Call(func() (bool, error) {
 			_, err := f.c.DeleteBucketWithContext(ctx, &req)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err == nil {
 			fs.Infof(f, "Bucket %q deleted", bucket)
@@ -2242,7 +2241,7 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
 	}
 	return f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CopyObjectWithContext(ctx, req)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 }
 
@@ -2286,7 +2285,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
 	if err := f.pacer.Call(func() (bool, error) {
 		var err error
 		cout, err = f.c.CreateMultipartUploadWithContext(ctx, req)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	}); err != nil {
 		return err
 	}
@@ -2302,7 +2301,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
 				UploadId:     uid,
 				RequestPayer: req.RequestPayer,
 			})
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 	})()
 
@@ -2325,7 +2324,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
 			uploadPartReq.CopySourceRange = aws.String(calculateRange(partSize, partNum-1, numParts, srcSize))
 			uout, err := f.c.UploadPartCopyWithContext(ctx, uploadPartReq)
 			if err != nil {
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			}
 			parts = append(parts, &s3.CompletedPart{
 				PartNumber: &partNum,
@@ -2347,7 +2346,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
 			RequestPayer: req.RequestPayer,
 			UploadId:     uid,
 		})
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 }
 
@@ -2578,7 +2577,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			reqCopy.Key = &bucketPath
 			err = f.pacer.Call(func() (bool, error) {
 				_, err = f.c.RestoreObject(&reqCopy)
-				return f.shouldRetry(err)
+				return f.shouldRetry(ctx, err)
 			})
 			if err != nil {
 				st.Status = err.Error()
@@ -2626,7 +2625,7 @@ func (f *Fs) listMultipartUploads(ctx context.Context, bucket, key string) (uplo
 		var resp *s3.ListMultipartUploadsOutput
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListMultipartUploads(&req)
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return nil, errors.Wrapf(err, "list multipart uploads bucket %q key %q", bucket, key)
@@ -2801,7 +2800,7 @@ func (o *Object) headObject(ctx context.Context) (resp *s3.HeadObjectOutput, err
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObjectWithContext(ctx, &req)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -2957,7 +2956,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		var err error
 		httpReq.HTTPRequest = httpReq.HTTPRequest.WithContext(ctx)
 		err = httpReq.Send()
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
@@ -3016,7 +3015,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	err = f.pacer.Call(func() (bool, error) {
 		var err error
 		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &mReq)
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "multipart upload failed to initialise")
@@ -3035,7 +3034,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 				UploadId:     uid,
 				RequestPayer: req.RequestPayer,
 			})
-			return f.shouldRetry(err)
+			return f.shouldRetry(ctx, err)
 		})
 		if errCancel != nil {
 			fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
@@ -3111,7 +3110,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 				uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
 				if err != nil {
 					if partNum <= int64(concurrency) {
-						return f.shouldRetry(err)
+						return f.shouldRetry(ctx, err)
 					}
 					// retry all chunks once have done the first batch
 					return true, err
@@ -3151,7 +3150,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 			RequestPayer: req.RequestPayer,
 			UploadId:     uid,
 		})
-		return f.shouldRetry(err)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "multipart upload failed to finalise")
@@ -3306,11 +3305,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			var err error
 			resp, err = o.fs.srv.Do(httpReq)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return o.fs.shouldRetry(ctx, err)
 			}
 			body, err := rest.ReadBody(resp)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return o.fs.shouldRetry(ctx, err)
 			}
 			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
 				return false, nil
@@ -3361,7 +3360,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.DeleteObjectWithContext(ctx, &req)
-		return o.fs.shouldRetry(err)
+		return o.fs.shouldRetry(ctx, err)
 	})
 	return err
 }

backend/seafile/seafile.go

@@ -296,36 +296,32 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // Config callback for 2FA
-func Config(ctx context.Context, name string, m configmap.Mapper) {
+func Config(ctx context.Context, name string, m configmap.Mapper) error {
 	ci := fs.GetConfig(ctx)
 	serverURL, ok := m.Get(configURL)
 	if !ok || serverURL == "" {
 		// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
-		fmt.Print("\nOperation not supported on this remote.\nIf you need a 2FA code on your account, use the command:\n\nrclone config reconnect <remote name>:\n\n")
-		return
+		return errors.New("operation not supported on this remote. If you need a 2FA code on your account, use the command: nrclone config reconnect <remote name>: ")
 	}
 
 	// Stop if we are running non-interactive config
 	if ci.AutoConfirm {
-		return
+		return nil
 	}
 
 	u, err := url.Parse(serverURL)
 	if err != nil {
-		fs.Errorf(nil, "Invalid server URL %s", serverURL)
-		return
+		return errors.Errorf("invalid server URL %s", serverURL)
 	}
 
 	is2faEnabled, _ := m.Get(config2FA)
 	if is2faEnabled != "true" {
-		fmt.Println("Two-factor authentication is not enabled on this account.")
-		return
+		return errors.New("two-factor authentication is not enabled on this account")
 	}
 
 	username, _ := m.Get(configUser)
 	if username == "" {
-		fs.Errorf(nil, "A username is required")
-		return
+		return errors.New("a username is required")
 	}
 
 	password, _ := m.Get(configPassword)
@@ -372,11 +368,11 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
 			m.Set(configAuthToken, token)
 			// And delete any previous entry for password
 			m.Set(configPassword, "")
-			config.SaveConfig()
 			// And we're done here
 			break
 		}
 	}
+	return nil
 }
 
 // sets the AuthorizationToken up
@@ -408,7 +404,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// For 429 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.

backend/seafile/webapi.go

@@ -86,7 +86,7 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -112,7 +112,7 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -139,7 +139,7 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -170,7 +170,7 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -197,7 +197,7 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -228,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -271,7 +271,7 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -316,7 +316,7 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -358,7 +358,7 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -398,7 +398,7 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -438,7 +438,7 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -474,7 +474,7 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -505,7 +505,7 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -539,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to delete file")
@@ -565,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -614,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -659,7 +659,7 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -682,7 +682,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
 		"need_idx_progress": {"true"},
 		"replace":           {"1"},
 	}
-	formReader, contentType, _, err := rest.MultipartUpload(in, parameters, "file", f.opt.Enc.FromStandardName(filename))
+	formReader, contentType, _, err := rest.MultipartUpload(ctx, in, parameters, "file", f.opt.Enc.FromStandardName(filename))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make multipart upload")
 	}
@@ -739,7 +739,7 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -777,7 +777,7 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -818,7 +818,7 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -860,7 +860,7 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -900,7 +900,7 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -938,7 +938,7 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -976,7 +976,7 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -1030,7 +1030,7 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -1075,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil {

backend/sftp/sftp.go

@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -204,6 +205,36 @@ Fstat instead of Stat which is called on an already open file handle.
 It has been found that this helps with IBM Sterling SFTP servers which have
 "extractability" level set to 1 which means only 1 file can be opened at
 any given time.
+`,
+			Advanced: true,
+		}, {
+			Name:    "disable_concurrent_reads",
+			Default: false,
+			Help: `If set don't use concurrent reads
+
+Normally concurrent reads are safe to use and not using them will
+degrade performance, so this option is disabled by default.
+
+Some servers limit the amount number of times a file can be
+downloaded. Using concurrent reads can trigger this limit, so if you
+have a server which returns
+
+    Failed to copy: file does not exist
+
+Then you may need to enable this flag.
+
+If concurrent reads are disabled, the use_fstat option is ignored.
+`,
+			Advanced: true,
+		}, {
+			Name:    "disable_concurrent_writes",
+			Default: false,
+			Help: `If set don't use concurrent writes
+
+Normally rclone uses concurrent writes to upload files. This improves
+the performance greatly, especially for distant servers.
+
+This option disables concurrent writes should that be necessary.
 `,
 			Advanced: true,
 		}, {
@@ -224,28 +255,30 @@ Set to 0 to keep connections indefinitely.
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string      `config:"host"`
-	User              string      `config:"user"`
-	Port              string      `config:"port"`
-	Pass              string      `config:"pass"`
-	KeyPem            string      `config:"key_pem"`
-	KeyFile           string      `config:"key_file"`
-	KeyFilePass       string      `config:"key_file_pass"`
-	PubKeyFile        string      `config:"pubkey_file"`
-	KnownHostsFile    string      `config:"known_hosts_file"`
-	KeyUseAgent       bool        `config:"key_use_agent"`
-	UseInsecureCipher bool        `config:"use_insecure_cipher"`
-	DisableHashCheck  bool        `config:"disable_hashcheck"`
-	AskPassword       bool        `config:"ask_password"`
-	PathOverride      string      `config:"path_override"`
-	SetModTime        bool        `config:"set_modtime"`
-	Md5sumCommand     string      `config:"md5sum_command"`
-	Sha1sumCommand    string      `config:"sha1sum_command"`
-	SkipLinks         bool        `config:"skip_links"`
-	Subsystem         string      `config:"subsystem"`
-	ServerCommand     string      `config:"server_command"`
-	UseFstat          bool        `config:"use_fstat"`
-	IdleTimeout       fs.Duration `config:"idle_timeout"`
+	Host                    string      `config:"host"`
+	User                    string      `config:"user"`
+	Port                    string      `config:"port"`
+	Pass                    string      `config:"pass"`
+	KeyPem                  string      `config:"key_pem"`
+	KeyFile                 string      `config:"key_file"`
+	KeyFilePass             string      `config:"key_file_pass"`
+	PubKeyFile              string      `config:"pubkey_file"`
+	KnownHostsFile          string      `config:"known_hosts_file"`
+	KeyUseAgent             bool        `config:"key_use_agent"`
+	UseInsecureCipher       bool        `config:"use_insecure_cipher"`
+	DisableHashCheck        bool        `config:"disable_hashcheck"`
+	AskPassword             bool        `config:"ask_password"`
+	PathOverride            string      `config:"path_override"`
+	SetModTime              bool        `config:"set_modtime"`
+	Md5sumCommand           string      `config:"md5sum_command"`
+	Sha1sumCommand          string      `config:"sha1sum_command"`
+	SkipLinks               bool        `config:"skip_links"`
+	Subsystem               string      `config:"subsystem"`
+	ServerCommand           string      `config:"server_command"`
+	UseFstat                bool        `config:"use_fstat"`
+	DisableConcurrentReads  bool        `config:"disable_concurrent_reads"`
+	DisableConcurrentWrites bool        `config:"disable_concurrent_writes"`
+	IdleTimeout             fs.Duration `config:"idle_timeout"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -266,6 +299,7 @@ type Fs struct {
 	drain        *time.Timer // used to drain the pool when we stop using the connections
 	pacer        *fs.Pacer   // pacer for operations
 	savedpswd    string
+	transfers    int32 // count in use references
 }
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -328,6 +362,23 @@ func (c *conn) closed() error {
 	return nil
 }
 
+// Show that we are doing an upload or download
+//
+// Call removeTransfer() when done
+func (f *Fs) addTransfer() {
+	atomic.AddInt32(&f.transfers, 1)
+}
+
+// Show the upload or download done
+func (f *Fs) removeTransfer() {
+	atomic.AddInt32(&f.transfers, -1)
+}
+
+// getTransfers shows whether there are any transfers in progress
+func (f *Fs) getTransfers() int32 {
+	return atomic.LoadInt32(&f.transfers)
+}
+
 // Open a new connection to the SFTP server.
 func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
 	// Rate limit rate of new connections
@@ -373,7 +424,14 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
 		}
 	}
 	opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
-	opts = append(opts, sftp.UseFstat(f.opt.UseFstat))
+	opts = append(opts,
+		sftp.UseFstat(f.opt.UseFstat),
+		sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
+		sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites),
+	)
+	if f.opt.DisableConcurrentReads { // FIXME
+		fs.Errorf(f, "Ignoring disable_concurrent_reads after library reversion - see #5197")
+	}
 
 	return sftp.NewClientPipe(pr, pw, opts...)
 }
@@ -451,6 +509,13 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 func (f *Fs) drainPool(ctx context.Context) (err error) {
 	f.poolMu.Lock()
 	defer f.poolMu.Unlock()
+	if transfers := f.getTransfers(); transfers != 0 {
+		fs.Debugf(f, "Not closing %d unused connections as %d transfers in progress", len(f.pool), transfers)
+		if f.opt.IdleTimeout > 0 {
+			f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+		}
+		return nil
+	}
 	if f.opt.IdleTimeout > 0 {
 		f.drain.Stop()
 	}
@@ -1331,18 +1396,19 @@ func (o *Object) stat(ctx context.Context) error {
 //
 // it also updates the info field
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	if o.fs.opt.SetModTime {
-		c, err := o.fs.getSftpConnection(ctx)
-		if err != nil {
-			return errors.Wrap(err, "SetModTime")
-		}
-		err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
-		o.fs.putSftpConnection(&c, err)
-		if err != nil {
-			return errors.Wrap(err, "SetModTime failed")
-		}
+	if !o.fs.opt.SetModTime {
+		return nil
 	}
-	err := o.stat(ctx)
+	c, err := o.fs.getSftpConnection(ctx)
+	if err != nil {
+		return errors.Wrap(err, "SetModTime")
+	}
+	err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
+	o.fs.putSftpConnection(&c, err)
+	if err != nil {
+		return errors.Wrap(err, "SetModTime failed")
+	}
+	err = o.stat(ctx)
 	if err != nil {
 		return errors.Wrap(err, "SetModTime stat failed")
 	}
@@ -1356,18 +1422,22 @@ func (o *Object) Storable() bool {
 
 // objectReader represents a file open for reading on the SFTP server
 type objectReader struct {
+	f          *Fs
 	sftpFile   *sftp.File
 	pipeReader *io.PipeReader
 	done       chan struct{}
 }
 
-func newObjectReader(sftpFile *sftp.File) *objectReader {
+func (f *Fs) newObjectReader(sftpFile *sftp.File) *objectReader {
 	pipeReader, pipeWriter := io.Pipe()
 	file := &objectReader{
+		f:          f,
 		sftpFile:   sftpFile,
 		pipeReader: pipeReader,
 		done:       make(chan struct{}),
 	}
+	// Show connection in use
+	f.addTransfer()
 
 	go func() {
 		// Use sftpFile.WriteTo to pump data so that it gets a
@@ -1397,6 +1467,8 @@ func (file *objectReader) Close() (err error) {
 	_ = file.pipeReader.Close()
 	// Wait for the background process to finish
 	<-file.done
+	// Show connection no longer in use
+	file.f.removeTransfer()
 	return err
 }
 
@@ -1430,12 +1502,27 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			return nil, errors.Wrap(err, "Open Seek failed")
 		}
 	}
-	in = readers.NewLimitedReadCloser(newObjectReader(sftpFile), limit)
+	in = readers.NewLimitedReadCloser(o.fs.newObjectReader(sftpFile), limit)
 	return in, nil
 }
 
+type sizeReader struct {
+	io.Reader
+	size int64
+}
+
+// Size returns the expected size of the stream
+//
+// It is used in sftpFile.ReadFrom as a hint to work out the
+// concurrency needed
+func (sr *sizeReader) Size() int64 {
+	return sr.size
+}
+
 // Update a remote sftp file using the data <in> and ModTime from <src>
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	o.fs.addTransfer() // Show transfer in progress
+	defer o.fs.removeTransfer()
 	// Clear the hash cache since we are about to update the object
 	o.md5sum = nil
 	o.sha1sum = nil
@@ -1463,7 +1550,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			fs.Debugf(src, "Removed after failed upload: %v", err)
 		}
 	}
-	_, err = file.ReadFrom(in)
+	_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
 	if err != nil {
 		remove()
 		return errors.Wrap(err, "Update ReadFrom failed")
@@ -1473,10 +1560,28 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		remove()
 		return errors.Wrap(err, "Update Close failed")
 	}
+
+	// Set the mod time - this stats the object if o.fs.opt.SetModTime == true
 	err = o.SetModTime(ctx, src.ModTime(ctx))
 	if err != nil {
 		return errors.Wrap(err, "Update SetModTime failed")
 	}
+
+	// Stat the file after the upload to read its stats back if o.fs.opt.SetModTime == false
+	if !o.fs.opt.SetModTime {
+		err = o.stat(ctx)
+		if err == fs.ErrorObjectNotFound {
+			// In the specific case of o.fs.opt.SetModTime == false
+			// if the object wasn't found then don't return an error
+			fs.Debugf(o, "Not found after upload with set_modtime=false so returning best guess")
+			o.modTime = src.ModTime(ctx)
+			o.size = src.Size()
+			o.mode = os.FileMode(0666) // regular file
+		} else if err != nil {
+			return errors.Wrap(err, "Update stat failed")
+		}
+	}
+
 	return nil
 }
 

backend/sharefile/sharefile.go

@@ -77,7 +77,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -136,7 +135,7 @@ func init() {
 		Name:        "sharefile",
 		Description: "Citrix Sharefile",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			oauthConfig := newOauthConfig("")
 			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
 				if auth == nil || auth.Form == nil {
@@ -157,8 +156,9 @@ func init() {
 			}
 			err := oauthutil.Config(ctx, "sharefile", name, m, oauthConfig, &opt)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: []fs.Option{{
 			Name:     "upload_cutoff",
@@ -299,7 +299,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -324,7 +327,7 @@ func (f *Fs) readMetaDataForIDPath(ctx context.Context, id, path string, directo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &item)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
@@ -631,7 +634,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &req, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "CreateDir")
@@ -663,7 +666,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -912,7 +915,7 @@ func (f *Fs) updateItem(ctx context.Context, id, leaf, directoryID string, modTi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1133,7 +1136,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1294,7 +1297,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var dl api.DownloadSpecification
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open: fetch download specification")
@@ -1309,7 +1312,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
@@ -1365,7 +1368,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload get specification")
@@ -1390,7 +1393,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var finish api.UploadFinishResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file")
@@ -1426,7 +1429,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "remove")

backend/sharefile/upload.go

@@ -155,7 +155,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		resp, err := up.f.srv.Call(ctx, &opts)
 		if err != nil {
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		}
 		respBody, err = rest.ReadBody(resp)
 		// retry all errors now that the multipart upload has started

backend/sugarsync/sugarsync.go

@@ -16,7 +16,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -76,17 +75,17 @@ func init() {
 		Name:        "sugarsync",
 		Description: "Sugarsync",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			opt := new(Options)
 			err := configstruct.Set(m, opt)
 			if err != nil {
-				log.Fatalf("Failed to read options: %v", err)
+				return errors.Wrap(err, "failed to read options")
 			}
 
 			if opt.RefreshToken != "" {
 				fmt.Printf("Already have a token - refresh?\n")
 				if !config.ConfirmWithConfig(ctx, m, "config_refresh_token", true) {
-					return
+					return nil
 				}
 			}
 			fmt.Printf("Username (email address)> ")
@@ -111,13 +110,14 @@ func init() {
 			// FIXME
 			//err = f.pacer.Call(func() (bool, error) {
 			resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
-			//	return shouldRetry(resp, err)
+			//	return shouldRetry(ctx, resp, err)
 			//})
 			if err != nil {
-				log.Fatalf("Failed to get token: %v", err)
+				return errors.Wrap(err, "failed to get token")
 			}
 			opt.RefreshToken = resp.Header.Get("Location")
 			m.Set("refresh_token", opt.RefreshToken)
+			return nil
 		},
 		Options: []fs.Option{{
 			Name: "app_id",
@@ -248,7 +248,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -288,7 +291,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
@@ -325,7 +328,7 @@ func (f *Fs) getAuthToken(ctx context.Context) error {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get authorization")
@@ -373,7 +376,7 @@ func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &user)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get user")
@@ -567,7 +570,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -618,7 +621,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -774,7 +777,7 @@ func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string,
 		}
 		return f.pacer.Call(func() (bool, error) {
 			resp, err := f.srv.Call(ctx, &opts)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 	}
 	// Move file/dir to deleted files if not hard delete
@@ -880,7 +883,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &copyFile, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -934,7 +937,7 @@ func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &move, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -964,7 +967,7 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err
 	var resp *http.Response
 	return f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &move, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 }
 
@@ -1053,7 +1056,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var info *api.File
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1182,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1204,7 +1207,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1262,7 +1265,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to upload file")

backend/swift/swift.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/ncw/swift/v2"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
@@ -291,7 +292,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// If this is a swift.Error object extract the HTTP error code
 	if swiftError, ok := err.(*swift.Error); ok {
 		for _, e := range retryErrorCodes {
@@ -307,7 +311,7 @@ func shouldRetry(err error) (bool, error) {
 // shouldRetryHeaders returns a boolean as to whether this err
 // deserves to be retried.  It reads the headers passed in looking for
 // `Retry-After`. It returns the err as a convenience
-func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
+func shouldRetryHeaders(ctx context.Context, headers swift.Headers, err error) (bool, error) {
 	if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
 		if value := headers["Retry-After"]; value != "" {
 			retryAfter, parseErr := strconv.Atoi(value)
@@ -326,7 +330,7 @@ func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
 			}
 		}
 	}
-	return shouldRetry(err)
+	return shouldRetry(ctx, err)
 }
 
 // parsePath parses a remote 'url'
@@ -468,7 +472,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c
 		err = f.pacer.Call(func() (bool, error) {
 			var rxHeaders swift.Headers
 			info, rxHeaders, err = f.c.Object(ctx, f.rootContainer, encodedDirectory)
-			return shouldRetryHeaders(rxHeaders, err)
+			return shouldRetryHeaders(ctx, rxHeaders, err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			newRoot := path.Dir(f.root)
@@ -576,7 +580,7 @@ func (f *Fs) listContainerRoot(ctx context.Context, container, directory, prefix
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			objects, err = f.c.Objects(ctx, container, opts)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			for i := range objects {
@@ -661,7 +665,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err
 	var containers []swift.Container
 	err = f.pacer.Call(func() (bool, error) {
 		containers, err = f.c.ContainersAll(ctx, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "container listing failed")
@@ -753,7 +757,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		containers, err = f.c.ContainersAll(ctx, nil)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "container listing failed")
@@ -805,7 +809,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 			err = f.pacer.Call(func() (bool, error) {
 				var rxHeaders swift.Headers
 				_, rxHeaders, err = f.c.Container(ctx, container)
-				return shouldRetryHeaders(rxHeaders, err)
+				return shouldRetryHeaders(ctx, rxHeaders, err)
 			})
 		}
 		if err == swift.ContainerNotFound {
@@ -815,7 +819,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 			}
 			err = f.pacer.Call(func() (bool, error) {
 				err = f.c.ContainerCreate(ctx, container, headers)
-				return shouldRetry(err)
+				return shouldRetry(ctx, err)
 			})
 			if err == nil {
 				fs.Infof(f, "Container %q created", container)
@@ -836,7 +840,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	err := f.cache.Remove(container, func() error {
 		err := f.pacer.Call(func() (bool, error) {
 			err := f.c.ContainerDelete(ctx, container)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 		if err == nil {
 			fs.Infof(f, "Container %q removed", container)
@@ -902,18 +906,125 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	srcContainer, srcPath := srcObj.split()
-	err = f.pacer.Call(func() (bool, error) {
-		var rxHeaders swift.Headers
-		rxHeaders, err = f.c.ObjectCopy(ctx, srcContainer, srcPath, dstContainer, dstPath, nil)
-		return shouldRetryHeaders(rxHeaders, err)
-	})
+	isLargeObject, err := srcObj.isLargeObject(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if isLargeObject {
+		/*handle large object*/
+		err = copyLargeObject(ctx, f, srcObj, dstContainer, dstPath)
+	} else {
+		srcContainer, srcPath := srcObj.split()
+		err = f.pacer.Call(func() (bool, error) {
+			var rxHeaders swift.Headers
+			rxHeaders, err = f.c.ObjectCopy(ctx, srcContainer, srcPath, dstContainer, dstPath, nil)
+			return shouldRetryHeaders(ctx, rxHeaders, err)
+		})
+	}
 	if err != nil {
 		return nil, err
 	}
 	return f.NewObject(ctx, remote)
 }
 
+func copyLargeObject(ctx context.Context, f *Fs, src *Object, dstContainer string, dstPath string) error {
+	segmentsContainer := dstContainer + "_segments"
+	err := f.makeContainer(ctx, segmentsContainer)
+	if err != nil {
+		return err
+	}
+	segments, err := src.getSegmentsLargeObject(ctx)
+	if err != nil {
+		return err
+	}
+	if len(segments) == 0 {
+		return errors.New("could not copy object, list segments are empty")
+	}
+	nanoSeconds := time.Now().Nanosecond()
+	prefixSegment := fmt.Sprintf("%v/%v/%s", nanoSeconds, src.size, strings.ReplaceAll(uuid.New().String(), "-", ""))
+	copiedSegmentsLen := 10
+	for _, value := range segments {
+		if len(value) <= 0 {
+			continue
+		}
+		fragment := value[0]
+		if len(fragment) <= 0 {
+			continue
+		}
+		copiedSegmentsLen = len(value)
+		firstIndex := strings.Index(fragment, "/")
+		if firstIndex < 0 {
+			firstIndex = 0
+		} else {
+			firstIndex = firstIndex + 1
+		}
+		lastIndex := strings.LastIndex(fragment, "/")
+		if lastIndex < 0 {
+			lastIndex = len(fragment)
+		} else {
+			lastIndex = lastIndex - 1
+		}
+		prefixSegment = fragment[firstIndex:lastIndex]
+		break
+	}
+	copiedSegments := make([]string, copiedSegmentsLen)
+	defer handleCopyFail(ctx, f, segmentsContainer, copiedSegments, err)
+	for c, ss := range segments {
+		if len(ss) <= 0 {
+			continue
+		}
+		for _, s := range ss {
+			lastIndex := strings.LastIndex(s, "/")
+			if lastIndex <= 0 {
+				lastIndex = 0
+			} else {
+				lastIndex = lastIndex + 1
+			}
+			segmentName := dstPath + "/" + prefixSegment + "/" + s[lastIndex:]
+			err = f.pacer.Call(func() (bool, error) {
+				var rxHeaders swift.Headers
+				rxHeaders, err = f.c.ObjectCopy(ctx, c, s, segmentsContainer, segmentName, nil)
+				copiedSegments = append(copiedSegments, segmentName)
+				return shouldRetryHeaders(ctx, rxHeaders, err)
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	m := swift.Metadata{}
+	headers := m.ObjectHeaders()
+	headers["X-Object-Manifest"] = urlEncode(fmt.Sprintf("%s/%s/%s", segmentsContainer, dstPath, prefixSegment))
+	headers["Content-Length"] = "0"
+	emptyReader := bytes.NewReader(nil)
+	err = f.pacer.Call(func() (bool, error) {
+		var rxHeaders swift.Headers
+		rxHeaders, err = f.c.ObjectPut(ctx, dstContainer, dstPath, emptyReader, true, "", src.contentType, headers)
+		return shouldRetryHeaders(ctx, rxHeaders, err)
+	})
+	return err
+}
+
+//remove copied segments when copy process failed
+func handleCopyFail(ctx context.Context, f *Fs, segmentsContainer string, segments []string, err error) {
+	fs.Debugf(f, "handle copy segment fail")
+	if err == nil {
+		return
+	}
+	if len(segmentsContainer) == 0 {
+		fs.Debugf(f, "invalid segments container")
+		return
+	}
+	if len(segments) == 0 {
+		fs.Debugf(f, "segments is empty")
+		return
+	}
+	fs.Debugf(f, "action delete segments what copied")
+	for _, v := range segments {
+		_ = f.c.ObjectDelete(ctx, segmentsContainer, v)
+	}
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
@@ -1041,7 +1152,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	container, containerPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, h, err = o.fs.c.Object(ctx, container, containerPath)
-		return shouldRetryHeaders(h, err)
+		return shouldRetryHeaders(ctx, h, err)
 	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
@@ -1100,7 +1211,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	container, containerPath := o.split()
 	return o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.c.ObjectUpdate(ctx, container, containerPath, newHeaders)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 }
 
@@ -1121,7 +1232,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var rxHeaders swift.Headers
 		in, rxHeaders, err = o.fs.c.ObjectOpen(ctx, container, containerPath, !isRanging, headers)
-		return shouldRetryHeaders(rxHeaders, err)
+		return shouldRetryHeaders(ctx, rxHeaders, err)
 	})
 	return
 }
@@ -1211,7 +1322,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var rxHeaders swift.Headers
 		_, rxHeaders, err = o.fs.c.Container(ctx, segmentsContainer)
-		return shouldRetryHeaders(rxHeaders, err)
+		return shouldRetryHeaders(ctx, rxHeaders, err)
 	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
@@ -1220,7 +1331,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
 		}
 		err = o.fs.pacer.Call(func() (bool, error) {
 			err = o.fs.c.ContainerCreate(ctx, segmentsContainer, headers)
-			return shouldRetry(err)
+			return shouldRetry(ctx, err)
 		})
 	}
 	if err != nil {
@@ -1241,7 +1352,8 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
 		if segmentInfos == nil || len(segmentInfos) == 0 {
 			return
 		}
-		deleteChunks(ctx, o, segmentsContainer, segmentInfos)
+		_ctx := context.Background()
+		deleteChunks(_ctx, o, segmentsContainer, segmentInfos)
 	})()
 	for {
 		// can we read at least one byte?
@@ -1267,7 +1379,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
 			if err == nil {
 				segmentInfos = append(segmentInfos, segmentPath)
 			}
-			return shouldRetryHeaders(rxHeaders, err)
+			return shouldRetryHeaders(ctx, rxHeaders, err)
 		})
 		if err != nil {
 			return "", err
@@ -1281,7 +1393,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var rxHeaders swift.Headers
 		rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, emptyReader, true, "", contentType, headers)
-		return shouldRetryHeaders(rxHeaders, err)
+		return shouldRetryHeaders(ctx, rxHeaders, err)
 	})
 
 	if err == nil {
@@ -1356,7 +1468,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		var rxHeaders swift.Headers
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, in, true, "", contentType, headers)
-			return shouldRetryHeaders(rxHeaders, err)
+			return shouldRetryHeaders(ctx, rxHeaders, err)
 		})
 		if err != nil {
 			return err
@@ -1414,7 +1526,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	// Remove file/manifest first
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.c.ObjectDelete(ctx, container, containerPath)
-		return shouldRetry(err)
+		return shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err

backend/swift/swift_internal_test.go

@@ -1,6 +1,7 @@
 package swift
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -32,6 +33,7 @@ func TestInternalUrlEncode(t *testing.T) {
 }
 
 func TestInternalShouldRetryHeaders(t *testing.T) {
+	ctx := context.Background()
 	headers := swift.Headers{
 		"Content-Length": "64",
 		"Content-Type":   "text/html; charset=UTF-8",
@@ -45,7 +47,7 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
 
 	// Short sleep should just do the sleep
 	start := time.Now()
-	retry, gotErr := shouldRetryHeaders(headers, err)
+	retry, gotErr := shouldRetryHeaders(ctx, headers, err)
 	dt := time.Since(start)
 	assert.True(t, retry)
 	assert.Equal(t, err, gotErr)
@@ -54,7 +56,7 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
 	// Long sleep should return RetryError
 	headers["Retry-After"] = "3600"
 	start = time.Now()
-	retry, gotErr = shouldRetryHeaders(headers, err)
+	retry, gotErr = shouldRetryHeaders(ctx, headers, err)
 	dt = time.Since(start)
 	assert.True(t, dt < time.Second)
 	assert.False(t, retry)

backend/swift/swift_test.go

@@ -80,6 +80,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("NoChunk", f.testNoChunk)
 	t.Run("WithChunk", f.testWithChunk)
 	t.Run("WithChunkFail", f.testWithChunkFail)
+	t.Run("CopyLargeObject", f.testCopyLargeObject)
 }
 
 func (f *Fs) testWithChunk(t *testing.T) {
@@ -154,4 +155,39 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
 	require.Empty(t, objs)
 }
 
+func (f *Fs) testCopyLargeObject(t *testing.T) {
+	preConfChunkSize := f.opt.ChunkSize
+	preConfChunk := f.opt.NoChunk
+	f.opt.NoChunk = false
+	f.opt.ChunkSize = 1024 * fs.Byte
+	defer func() {
+		//restore old config after test
+		f.opt.ChunkSize = preConfChunkSize
+		f.opt.NoChunk = preConfChunk
+	}()
+
+	file := fstest.Item{
+		ModTime: fstest.Time("2020-12-31T04:05:06.499999999Z"),
+		Path:    "large.txt",
+		Size:    -1, // use unknown size during upload
+	}
+	const contentSize = 2048
+	contents := random.String(contentSize)
+	buf := bytes.NewBufferString(contents)
+	uploadHash := hash.NewMultiHasher()
+	in := io.TeeReader(buf, uploadHash)
+
+	file.Size = -1
+	obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
+	ctx := context.TODO()
+	obj, err := f.Features().PutStream(ctx, in, obji)
+	require.NoError(t, err)
+	require.NotEmpty(t, obj)
+	remoteTarget := "large.txt (copy)"
+	objTarget, err := f.Features().Copy(ctx, obj, remoteTarget)
+	require.NoError(t, err)
+	require.NotEmpty(t, objTarget)
+	require.Equal(t, obj.Size(), objTarget.Size())
+}
+
 var _ fstests.InternalTester = (*Fs)(nil)

backend/tardigrade/fs.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"path"
 	"strings"
 	"time"
@@ -42,7 +41,7 @@ func init() {
 		Name:        "tardigrade",
 		Description: "Tardigrade Decentralized Cloud Storage",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, configMapper configmap.Mapper) {
+		Config: func(ctx context.Context, name string, configMapper configmap.Mapper) error {
 			provider, _ := configMapper.Get(fs.ConfigProvider)
 
 			config.FileDeleteKey(name, fs.ConfigProvider)
@@ -54,7 +53,7 @@ func init() {
 
 				// satelliteString contains always default and passphrase can be empty
 				if apiKey == "" {
-					return
+					return nil
 				}
 
 				satellite, found := satMap[satelliteString]
@@ -64,12 +63,12 @@ func init() {
 
 				access, err := uplink.RequestAccessWithPassphrase(context.TODO(), satellite, apiKey, passphrase)
 				if err != nil {
-					log.Fatalf("Couldn't create access grant: %v", err)
+					return errors.Wrap(err, "couldn't create access grant")
 				}
 
 				serializedAccess, err := access.Serialize()
 				if err != nil {
-					log.Fatalf("Couldn't serialize access grant: %v", err)
+					return errors.Wrap(err, "couldn't serialize access grant")
 				}
 				configMapper.Set("satellite_address", satellite)
 				configMapper.Set("access_grant", serializedAccess)
@@ -78,8 +77,9 @@ func init() {
 				config.FileDeleteKey(name, "api_key")
 				config.FileDeleteKey(name, "passphrase")
 			} else {
-				log.Fatalf("Invalid provider type: %s", provider)
+				return errors.Errorf("invalid provider type: %s", provider)
 			}
+			return nil
 		},
 		Options: []fs.Option{
 			{

backend/union/entry.go

@@ -59,7 +59,17 @@ func (d *Directory) candidates() []upstream.Entry {
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	entries, err := o.fs.actionEntries(o.candidates()...)
-	if err != nil {
+	if err == fs.ErrorPermissionDenied {
+		// There are no candidates in this object which can be written to
+		// So attempt to create a new object instead
+		newO, err := o.fs.put(ctx, in, src, false, options...)
+		if err != nil {
+			return err
+		}
+		// Update current object
+		*o = *newO.(*Object)
+		return nil
+	} else if err != nil {
 		return err
 	}
 	if len(entries) == 1 {

backend/union/policy/epff.go

@@ -17,7 +17,9 @@ func init() {
 type EpFF struct{}
 
 func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath string) (*upstream.Fs, error) {
-	ch := make(chan *upstream.Fs)
+	ch := make(chan *upstream.Fs, len(upstreams))
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for _, u := range upstreams {
 		u := u // Closure
 		go func() {
@@ -30,16 +32,10 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath stri
 		}()
 	}
 	var u *upstream.Fs
-	for i := 0; i < len(upstreams); i++ {
+	for range upstreams {
 		u = <-ch
 		if u != nil {
-			// close remaining goroutines
-			go func(num int) {
-				defer close(ch)
-				for i := 0; i < num; i++ {
-					<-ch
-				}
-			}(len(upstreams) - 1 - i)
+			break
 		}
 	}
 	if u == nil {

backend/union/union_internal_test.go

@@ -0,0 +1,67 @@
+package union
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func (f *Fs) TestInternalReadOnly(t *testing.T) {
+	if f.name != "TestUnionRO" {
+		t.Skip("Only on RO union")
+	}
+	dir := "TestInternalReadOnly"
+	ctx := context.Background()
+	rofs := f.upstreams[len(f.upstreams)-1]
+	assert.False(t, rofs.IsWritable())
+
+	// Put a file onto the read only fs
+	contents := random.String(50)
+	file1 := fstest.NewItem(dir+"/file.txt", contents, time.Now())
+	_, obj1 := fstests.PutTestContents(ctx, t, rofs, &file1, contents, true)
+
+	// Check read from readonly fs via union
+	o, err := f.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	assert.Equal(t, int64(50), o.Size())
+
+	// Now call Update on the union Object with new data
+	contents2 := random.String(100)
+	file2 := fstest.NewItem(dir+"/file.txt", contents2, time.Now())
+	in := bytes.NewBufferString(contents2)
+	src := object.NewStaticObjectInfo(file2.Path, file2.ModTime, file2.Size, true, nil, nil)
+	err = o.Update(ctx, in, src)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), o.Size())
+
+	// Check we read the new object via the union
+	o, err = f.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), o.Size())
+
+	// Remove the object
+	assert.NoError(t, o.Remove(ctx))
+
+	// Check we read the old object in the read only layer now
+	o, err = f.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	assert.Equal(t, int64(50), o.Size())
+
+	// Remove file and dir from read only fs
+	assert.NoError(t, obj1.Remove(ctx))
+	assert.NoError(t, rofs.Rmdir(ctx, dir))
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("ReadOnly", f.TestInternalReadOnly)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/union/union_test.go

@@ -2,13 +2,15 @@
 package union_test
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -24,17 +26,28 @@ func TestIntegration(t *testing.T) {
 	})
 }
 
+func makeTestDirs(t *testing.T, n int) (dirs []string, clean func()) {
+	for i := 1; i <= n; i++ {
+		dir, err := ioutil.TempDir("", fmt.Sprintf("rclone-union-test-%d", n))
+		require.NoError(t, err)
+		dirs = append(dirs, dir)
+	}
+	clean = func() {
+		for _, dir := range dirs {
+			err := os.RemoveAll(dir)
+			assert.NoError(t, err)
+		}
+	}
+	return dirs, clean
+}
+
 func TestStandard(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-standard1")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-standard2")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-standard3")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
 	name := "TestUnion"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",
@@ -54,13 +67,9 @@ func TestRO(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-ro1")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-ro2")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-ro3")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + ":ro " + tempdir3 + ":ro"
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + ":ro " + dirs[2] + ":ro"
 	name := "TestUnionRO"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",
@@ -80,13 +89,9 @@ func TestNC(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-nc1")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-nc2")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-nc3")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + ":nc " + tempdir3 + ":nc"
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + ":nc " + dirs[2] + ":nc"
 	name := "TestUnionNC"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",
@@ -106,13 +111,9 @@ func TestPolicy1(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy11")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy12")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy13")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
 	name := "TestUnionPolicy1"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",
@@ -132,13 +133,9 @@ func TestPolicy2(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy21")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy22")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy23")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
 	name := "TestUnionPolicy2"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",
@@ -158,13 +155,9 @@ func TestPolicy3(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy31")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy32")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy33")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
+	dirs, clean := makeTestDirs(t, 3)
+	defer clean()
+	upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
 	name := "TestUnionPolicy3"
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: name + ":",

backend/union/upstream/upstream.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/fspath"
 )
 
 var (
@@ -62,7 +63,7 @@ type Entry interface {
 // New creates a new Fs based on the
 // string formatted `type:root_path(:ro/:nc)`
 func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs, error) {
-	_, configName, fsPath, err := fs.ParseRemote(remote)
+	configName, fsPath, err := fspath.SplitFs(remote)
 	if err != nil {
 		return nil, err
 	}
@@ -83,15 +84,13 @@ func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs
 		f.creatable = false
 		fsPath = fsPath[0 : len(fsPath)-3]
 	}
-	if configName != "local" {
-		fsPath = configName + ":" + fsPath
-	}
-	rFs, err := cache.Get(ctx, fsPath)
+	remote = configName + fsPath
+	rFs, err := cache.Get(ctx, remote)
 	if err != nil && err != fs.ErrorIsFile {
 		return nil, err
 	}
 	f.RootFs = rFs
-	rootString := path.Join(fsPath, filepath.ToSlash(root))
+	rootString := path.Join(remote, filepath.ToSlash(root))
 	myFs, err := cache.Get(ctx, rootString)
 	if err != nil && err != fs.ErrorIsFile {
 		return nil, err

backend/uptobox/api/types.go

@@ -0,0 +1,170 @@
+package api
+
+import "fmt"
+
+// Error contains the error code and message returned by the API
+type Error struct {
+	Success    bool   `json:"success,omitempty"`
+	StatusCode int    `json:"statusCode,omitempty"`
+	Message    string `json:"message,omitempty"`
+	Data       string `json:"data,omitempty"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e Error) Error() string {
+	out := fmt.Sprintf("api error %d", e.StatusCode)
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.Data != "" {
+		out += ": " + e.Data
+	}
+	return out
+}
+
+// FolderEntry represents a Uptobox subfolder when listing folder contents
+type FolderEntry struct {
+	FolderID    uint64 `json:"fld_id"`
+	Description string `json:"fld_descr"`
+	Password    string `json:"fld_password"`
+	FullPath    string `json:"fullPath"`
+	Path        string `json:"fld_name"`
+	Name        string `json:"name"`
+	Hash        string `json:"hash"`
+}
+
+// FolderInfo represents the current folder when listing folder contents
+type FolderInfo struct {
+	FolderID      uint64 `json:"fld_id"`
+	Hash          string `json:"hash"`
+	FileCount     uint64 `json:"fileCount"`
+	TotalFileSize int64  `json:"totalFileSize"`
+}
+
+// FileInfo represents a file when listing folder contents
+type FileInfo struct {
+	Name         string `json:"file_name"`
+	Description  string `json:"file_descr"`
+	Created      string `json:"file_created"`
+	Size         int64  `json:"file_size"`
+	Downloads    uint64 `json:"file_downloads"`
+	Code         string `json:"file_code"`
+	Password     string `json:"file_password"`
+	Public       int    `json:"file_public"`
+	LastDownload string `json:"file_last_download"`
+	ID           uint64 `json:"id"`
+}
+
+// ReadMetadataResponse is the response when listing folder contents
+type ReadMetadataResponse struct {
+	StatusCode int    `json:"statusCode"`
+	Message    string `json:"message"`
+	Data       struct {
+		CurrentFolder  FolderInfo    `json:"currentFolder"`
+		Folders        []FolderEntry `json:"folders"`
+		Files          []FileInfo    `json:"files"`
+		PageCount      int           `json:"pageCount"`
+		TotalFileCount int           `json:"totalFileCount"`
+		TotalFileSize  int64         `json:"totalFileSize"`
+	} `json:"data"`
+}
+
+// UploadInfo is the response when initiating an upload
+type UploadInfo struct {
+	StatusCode int    `json:"statusCode"`
+	Message    string `json:"message"`
+	Data       struct {
+		UploadLink string `json:"uploadLink"`
+		MaxUpload  string `json:"maxUpload"`
+	} `json:"data"`
+}
+
+// UploadResponse is the respnse to a successful upload
+type UploadResponse struct {
+	Files []struct {
+		Name      string `json:"name"`
+		Size      int64  `json:"size"`
+		URL       string `json:"url"`
+		DeleteURL string `json:"deleteUrl"`
+	} `json:"files"`
+}
+
+// UpdateResponse is a generic response to various action on files (rename/copy/move)
+type UpdateResponse struct {
+	Message    string `json:"message"`
+	StatusCode int    `json:"statusCode"`
+}
+
+// Download is the response when requesting a download link
+type Download struct {
+	StatusCode int    `json:"statusCode"`
+	Message    string `json:"message"`
+	Data       struct {
+		DownloadLink string `json:"dlLink"`
+	} `json:"data"`
+}
+
+// MetadataRequestOptions represents all the options when listing folder contents
+type MetadataRequestOptions struct {
+	Limit       uint64
+	Offset      uint64
+	SearchField string
+	Search      string
+}
+
+// CreateFolderRequest is used for creating a folder
+type CreateFolderRequest struct {
+	Token string `json:"token"`
+	Path  string `json:"path"`
+	Name  string `json:"name"`
+}
+
+// DeleteFolderRequest is used for deleting a folder
+type DeleteFolderRequest struct {
+	Token    string `json:"token"`
+	FolderID uint64 `json:"fld_id"`
+}
+
+// CopyMoveFileRequest is used for moving/copying a file
+type CopyMoveFileRequest struct {
+	Token               string `json:"token"`
+	FileCodes           string `json:"file_codes"`
+	DestinationFolderID uint64 `json:"destination_fld_id"`
+	Action              string `json:"action"`
+}
+
+// MoveFolderRequest is used for moving a folder
+type MoveFolderRequest struct {
+	Token               string `json:"token"`
+	FolderID            uint64 `json:"fld_id"`
+	DestinationFolderID uint64 `json:"destination_fld_id"`
+	Action              string `json:"action"`
+}
+
+// RenameFolderRequest is used for renaming a folder
+type RenameFolderRequest struct {
+	Token    string `json:"token"`
+	FolderID uint64 `json:"fld_id"`
+	NewName  string `json:"new_name"`
+}
+
+// UpdateFileInformation is used for renaming a file
+type UpdateFileInformation struct {
+	Token       string `json:"token"`
+	FileCode    string `json:"file_code"`
+	NewName     string `json:"new_name,omitempty"`
+	Description string `json:"description,omitempty"`
+	Password    string `json:"password,omitempty"`
+	Public      string `json:"public,omitempty"`
+}
+
+// RemoveFileRequest is used for deleting a file
+type RemoveFileRequest struct {
+	Token     string `json:"token"`
+	FileCodes string `json:"file_codes"`
+}
+
+// Token represents the authentication token
+type Token struct {
+	Token string `json:"token"`
+}

backend/uptobox/uptobox_test.go

@@ -0,0 +1,21 @@
+// Test Uptobox filesystem interface
+package uptobox_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/uptobox"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	if *fstest.RemoteName == "" {
+		*fstest.RemoteName = "TestUptobox:"
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: *fstest.RemoteName,
+		NilObject:  (*uptobox.Object)(nil),
+	})
+}

backend/webdav/odrvcookie/fetch.go

@@ -125,7 +125,7 @@ func (ca *CookieAuth) getSPCookie(conf *SharepointSuccessResponse) (*CookieRespo
 		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
 	}
 
-	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
+	u, err := url.Parse(spRoot.Scheme + "://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error while constructing login URL")
 	}

backend/webdav/webdav.go

@@ -196,7 +196,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// If we have a bearer token command and it has expired then refresh it
 	if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
 		fs.Debugf(f, "Bearer token expired: %v", err)
@@ -270,7 +273,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// does not exist
@@ -628,7 +631,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -800,7 +803,7 @@ func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	return err == nil
 }
@@ -822,7 +825,7 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// Check if it already exists. The response code for this isn't
@@ -911,7 +914,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -974,7 +977,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "Copy call failed")
@@ -1070,7 +1073,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "DirMove MOVE call failed")
@@ -1112,7 +1115,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
-		return f.shouldRetry(resp, err)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about call failed")
@@ -1240,7 +1243,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1291,7 +1294,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// Give the WebDAV server a chance to get its internal state in order after the
@@ -1318,7 +1321,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(resp, err)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 }
 

backend/yandex/yandex.go

@@ -60,12 +60,12 @@ func init() {
 		Name:        "yandex",
 		Description: "Yandex Disk",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			err := oauthutil.Config(ctx, "yandex", name, m, oauthConfig, nil)
 			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
-				return
+				return errors.Wrap(err, "failed to configure token")
 			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     config.ConfigEncoding,
@@ -153,7 +153,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -226,7 +229,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -248,22 +251,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	token, err := oauthutil.GetToken(name, m)
 	if err != nil {
-		log.Fatalf("Couldn't read OAuth token (this should never happen).")
+		return nil, errors.Wrap(err, "couldn't read OAuth token")
 	}
 	if token.RefreshToken == "" {
-		log.Fatalf("Unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend.")
+		return nil, errors.New("unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
 	}
 	if token.TokenType != "OAuth" {
 		token.TokenType = "OAuth"
 		err = oauthutil.PutToken(name, m, token, false)
 		if err != nil {
-			log.Fatalf("Couldn't save OAuth token (this should never happen).")
+			return nil, errors.Wrap(err, "couldn't save OAuth token")
 		}
 		log.Printf("Automatically upgraded OAuth config.")
 	}
 	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure Yandex: %v", err)
+		return nil, errors.Wrap(err, "failed to configure Yandex")
 	}
 
 	ci := fs.GetConfig(ctx)
@@ -468,7 +471,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		// fmt.Printf("CreateDir %q Error: %s\n", path, err.Error())
@@ -543,6 +546,9 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
 		var body []byte
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.Call(ctx, &opts)
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			if err != nil {
 				return fserrors.ShouldRetry(err), err
 			}
@@ -585,6 +591,9 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro
 	var body []byte
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
+		if fserrors.ContextError(ctx, &err) {
+			return false, err
+		}
 		if err != nil {
 			return fserrors.ShouldRetry(err), err
 		}
@@ -658,6 +667,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
 	var body []byte
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
+		if fserrors.ContextError(ctx, &err) {
+			return false, err
+		}
 		if err != nil {
 			return fserrors.ShouldRetry(err), err
 		}
@@ -810,7 +822,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if apiErr, ok := err.(*api.ErrorResponse); ok {
@@ -848,7 +860,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }
@@ -865,7 +877,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -999,7 +1011,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	return err
 }
@@ -1032,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -1047,7 +1059,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1071,7 +1083,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	if err != nil {
@@ -1089,7 +1101,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 
 	return err

backend/zoho/zoho.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -36,8 +35,8 @@ import (
 )
 
 const (
-	rcloneClientID              = "1000.OZNFWW075EKDSIE1R42HI9I2SUPC9A"
-	rcloneEncryptedClientSecret = "rn7myzbsYK3WlqO2EU6jU8wmj0ylsx7_1B5wvSaVncYbu1Wt0QxPW9FFbidjqAZtyxnBenYIWq1pcA"
+	rcloneClientID              = "1000.46MXF275FM2XV7QCHX5A7K3LGME66B"
+	rcloneEncryptedClientSecret = "U-2gxclZQBcOG9NPhjiXAhj-f0uQ137D0zar8YyNHXHkQZlTeSpIOQfmCb4oSpvosJp_SJLXmLLeUA"
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
@@ -73,32 +72,41 @@ func init() {
 		Name:        "zoho",
 		Description: "Zoho",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) error {
 			// Need to setup region before configuring oauth
-			setupRegion(m)
+			err := setupRegion(m)
+			if err != nil {
+				return err
+			}
 			opt := oauthutil.Options{
 				// No refresh token unless ApprovalForce is set
 				OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
 			}
 			if err := oauthutil.Config(ctx, "zoho", name, m, oauthConfig, &opt); err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+				return errors.Wrap(err, "failed to configure token")
 			}
 			// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
 			// it's own custom type
 			token, err := oauthutil.GetToken(name, m)
 			if err != nil {
-				log.Fatalf("Failed to read token: %v", err)
+				return errors.Wrap(err, "failed to read token")
 			}
 			if token.TokenType != "Zoho-oauthtoken" {
 				token.TokenType = "Zoho-oauthtoken"
 				err = oauthutil.PutToken(name, m, token, false)
 				if err != nil {
-					log.Fatalf("Failed to configure token: %v", err)
+					return errors.Wrap(err, "failed to configure token")
 				}
 			}
-			if err = setupRoot(ctx, name, m); err != nil {
-				log.Fatalf("Failed to configure root directory: %v", err)
+
+			if fs.GetConfig(ctx).AutoConfirm {
+				return nil
 			}
+
+			if err = setupRoot(ctx, name, m); err != nil {
+				return errors.Wrap(err, "failed to configure root directory")
+			}
+			return nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "region",
@@ -159,15 +167,16 @@ type Object struct {
 
 // ------------------------------------------------------------
 
-func setupRegion(m configmap.Mapper) {
+func setupRegion(m configmap.Mapper) error {
 	region, ok := m.Get("region")
-	if !ok {
-		log.Fatalf("No region set\n")
+	if !ok || region == "" {
+		return errors.New("no region set")
 	}
 	rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region)
 	accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
 	oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
 	oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
+	return nil
 }
 
 // ------------------------------------------------------------
@@ -203,7 +212,7 @@ func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api
 func setupRoot(ctx context.Context, name string, m configmap.Mapper) error {
 	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 	if err != nil {
-		log.Fatalf("Failed to load oAuthClient: %s", err)
+		return errors.Wrap(err, "failed to load oAuthClient")
 	}
 	authSrv := rest.NewClient(oAuthClient).SetRoot(accountsURL)
 	opts := rest.Opts{
@@ -257,7 +266,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
@@ -354,7 +366,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -369,6 +381,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err := configstruct.Set(m, opt); err != nil {
 		return nil, err
 	}
+	err := setupRegion(m)
+	if err != nil {
+		return nil, err
+	}
 
 	root = parsePath(root)
 	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
@@ -450,7 +466,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(resp, err)
+			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -555,7 +571,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -643,7 +659,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
 	params.Set("filename", name)
 	params.Set("parent_id", parent)
 	params.Set("override-name-exist", strconv.FormatBool(true))
-	formReader, contentType, overhead, err := rest.MultipartUpload(in, nil, "content", name)
+	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make multipart upload")
 	}
@@ -664,7 +680,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
 	var uploadResponse *api.UploadResponse
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "upload error")
@@ -746,7 +762,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) (err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "delete object failed")
@@ -816,7 +832,7 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e
 	var result *api.ItemInfo
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "rename failed")
@@ -869,7 +885,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var result *api.ItemList
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't copy file")
@@ -914,7 +930,7 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item,
 	var result *api.ItemList
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
@@ -1181,7 +1197,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err

bin/make_manual.py

@@ -62,6 +62,7 @@ docs = [
     "sftp.md",
     "sugarsync.md",
     "tardigrade.md",
+    "uptobox.md",
     "union.md",
     "webdav.md",
     "yandex.md",

cmd/about/about.go

@@ -44,10 +44,10 @@ var commandDefinition = &cobra.Command{
 	Use:   "about remote:",
 	Short: `Get quota information from the remote.`,
 	Long: `
-` + "`rclone about`" + `prints quota information about a remote to standard
+` + "`rclone about`" + ` prints quota information about a remote to standard
 output. The output is typically used, free, quota and trash contents.
 
-E.g. Typical output from` + "`rclone about remote:`" + `is:
+E.g. Typical output from ` + "`rclone about remote:`" + ` is:
 
     Total:   17G
     Used:    7.444G
@@ -75,7 +75,7 @@ Applying a ` + "`--full`" + ` flag to the command prints the bytes in full, e.g.
     Trashed: 104857602
     Other:   8849156022
 
-A ` + "`--json`" + `flag generates conveniently computer readable output, e.g.
+A ` + "`--json`" + ` flag generates conveniently computer readable output, e.g.
 
     {
         "total": 18253611008,

cmd/all/all.go

@@ -47,12 +47,14 @@ import (
 	_ "github.com/rclone/rclone/cmd/reveal"
 	_ "github.com/rclone/rclone/cmd/rmdir"
 	_ "github.com/rclone/rclone/cmd/rmdirs"
+	_ "github.com/rclone/rclone/cmd/selfupdate"
 	_ "github.com/rclone/rclone/cmd/serve"
 	_ "github.com/rclone/rclone/cmd/settier"
 	_ "github.com/rclone/rclone/cmd/sha1sum"
 	_ "github.com/rclone/rclone/cmd/size"
 	_ "github.com/rclone/rclone/cmd/sync"
 	_ "github.com/rclone/rclone/cmd/test"
+	_ "github.com/rclone/rclone/cmd/test/changenotify"
 	_ "github.com/rclone/rclone/cmd/test/histogram"
 	_ "github.com/rclone/rclone/cmd/test/info"
 	_ "github.com/rclone/rclone/cmd/test/makefiles"

cmd/authorize/authorize.go

@@ -29,8 +29,8 @@ rclone config.
 
 Use the --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.`,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 3, command, args)
-		config.Authorize(context.Background(), args, noAutoBrowser)
+		return config.Authorize(context.Background(), args, noAutoBrowser)
 	},
 }

cmd/cmd.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/config/configflags"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/filter"
@@ -35,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/buildinfo"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/terminal"
 	"github.com/spf13/cobra"
@@ -73,9 +75,24 @@ const (
 
 // ShowVersion prints the version to stdout
 func ShowVersion() {
+	osVersion, osKernel := buildinfo.GetOSVersion()
+	if osVersion == "" {
+		osVersion = "unknown"
+	}
+	if osKernel == "" {
+		osKernel = "unknown"
+	}
+
+	linking, tagString := buildinfo.GetLinkingAndTags()
+
 	fmt.Printf("rclone %s\n", fs.Version)
-	fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
-	fmt.Printf("- go version: %s\n", runtime.Version())
+	fmt.Printf("- os/version: %s\n", osVersion)
+	fmt.Printf("- os/kernel: %s\n", osKernel)
+	fmt.Printf("- os/type: %s\n", runtime.GOOS)
+	fmt.Printf("- os/arch: %s\n", runtime.GOARCH)
+	fmt.Printf("- go/version: %s\n", runtime.Version())
+	fmt.Printf("- go/linking: %s\n", linking)
+	fmt.Printf("- go/tags: %s\n", tagString)
 }
 
 // NewFsFile creates an Fs from a name but may point to a file.
@@ -83,7 +100,7 @@ func ShowVersion() {
 // It returns a string with the file name if points to a file
 // otherwise "".
 func NewFsFile(remote string) (fs.Fs, string) {
-	_, _, fsPath, err := fs.ParseRemote(remote)
+	_, fsPath, err := fspath.SplitFs(remote)
 	if err != nil {
 		err = fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
@@ -382,13 +399,22 @@ func initConfig() {
 	// Finish parsing any command line flags
 	configflags.SetFlags(ci)
 
+	// Load the config
+	err := configfile.LoadConfig(ctx)
+	if err != nil {
+		log.Fatalf("Failed to load config: %v", err)
+	}
+
+	// Start accounting
+	accounting.Start(ctx)
+
 	// Hide console window
 	if ci.NoConsole {
 		terminal.HideConsole()
 	}
 
 	// Load filters
-	err := filterflags.Reload(ctx)
+	err = filterflags.Reload(ctx)
 	if err != nil {
 		log.Fatalf("Failed to load filters: %v", err)
 	}
@@ -541,6 +567,9 @@ func Main() {
 	setupRootCommand(Root)
 	AddBackendFlags()
 	if err := Root.Execute(); err != nil {
+		if strings.HasPrefix(err.Error(), "unknown command") && selfupdateEnabled {
+			Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath())
+		}
 		log.Fatalf("Fatal error: %v", err)
 	}
 }

cmd/cmount/arch.go

@@ -0,0 +1,7 @@
+package cmount
+
+// ProvidedBy returns true if the rclone build for the given OS
+// provides support for lib/cgo-fuse
+func ProvidedBy(osName string) bool {
+	return osName == "windows" || osName == "darwin"
+}

cmd/cmount/mount.go

@@ -21,12 +21,13 @@ import (
 	"github.com/rclone/rclone/cmd/mountlib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/buildinfo"
 	"github.com/rclone/rclone/vfs"
 )
 
 func init() {
 	name := "cmount"
-	cmountOnly := runtime.GOOS == "windows" || runtime.GOOS == "darwin"
+	cmountOnly := ProvidedBy(runtime.GOOS)
 	if cmountOnly {
 		name = "mount"
 	}
@@ -35,6 +36,7 @@ func init() {
 		cmd.Aliases = append(cmd.Aliases, "cmount")
 	}
 	mountlib.AddRc("cmount", mount)
+	buildinfo.Tags = append(buildinfo.Tags, "cmount")
 }
 
 // Find the option string in the current options

cmd/config/config.go

@@ -22,6 +22,7 @@ func init() {
 	cmd.Root.AddCommand(configCommand)
 	configCommand.AddCommand(configEditCommand)
 	configCommand.AddCommand(configFileCommand)
+	configCommand.AddCommand(configTouchCommand)
 	configCommand.AddCommand(configShowCommand)
 	configCommand.AddCommand(configDumpCommand)
 	configCommand.AddCommand(configProvidersCommand)
@@ -41,9 +42,9 @@ var configCommand = &cobra.Command{
 remotes and manage existing ones. You may also set or remove a
 password to protect your configuration.
 `,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 0, command, args)
-		config.EditConfig(context.Background())
+		return config.EditConfig(context.Background())
 	},
 }
 
@@ -63,6 +64,15 @@ var configFileCommand = &cobra.Command{
 	},
 }
 
+var configTouchCommand = &cobra.Command{
+	Use:   "touch",
+	Short: `Ensure configuration file exists.`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 0, command, args)
+		config.SaveConfig()
+	},
+}
+
 var configShowCommand = &cobra.Command{
 	Use:   "show [<remote>]",
 	Short: `Print (decrypted) config file, or the config for a single remote.`,
@@ -262,8 +272,7 @@ This normally means going through the interactive oauth flow again.
 		if fsInfo.Config == nil {
 			return errors.Errorf("%s: doesn't support Reconnect", configName)
 		}
-		fsInfo.Config(ctx, configName, config)
-		return nil
+		return fsInfo.Config(ctx, configName, config)
 	},
 }
 

cmd/copyurl/copyurl.go

@@ -3,6 +3,7 @@ package copyurl
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 
 	"github.com/rclone/rclone/cmd"
@@ -13,15 +14,17 @@ import (
 )
 
 var (
-	autoFilename = false
-	stdout       = false
-	noClobber    = false
+	autoFilename  = false
+	printFilename = false
+	stdout        = false
+	noClobber     = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path")
+	flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename")
 	flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name")
 	flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file")
 }
@@ -33,15 +36,16 @@ var commandDefinition = &cobra.Command{
 Download a URL's content and copy it to the destination without saving
 it in temporary storage.
 
-Setting --auto-filename will cause the file name to be retrieved from
+Setting ` + "`--auto-filename`" + ` will cause the file name to be retrieved from
 the from URL (after any redirections) and used in the destination
-path.
+path. With ` + "`--print-filename`" + ` in addition, the resuling file name will
+be printed.
 
-Setting --no-clobber will prevent overwriting file on the 
+Setting ` + "`--no-clobber`" + ` will prevent overwriting file on the 
 destination if there is one with the same name.
 
-Setting --stdout or making the output file name "-" will cause the
-output to be written to standard output.
+Setting ` + "`--stdout`" + ` or making the output file name ` + "`-`" + `
+will cause the output to be written to standard output.
 `,
 	RunE: func(command *cobra.Command, args []string) (err error) {
 		cmd.CheckArgs(1, 2, command, args)
@@ -61,10 +65,14 @@ output to be written to standard output.
 			}
 		}
 		cmd.Run(true, true, command, func() error {
+			var dst fs.Object
 			if stdout {
 				err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
 			} else {
-				_, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber)
+				dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber)
+				if printFilename && err == nil && dst != nil {
+					fmt.Println(dst.Remote())
+				}
 			}
 			return err
 		})

cmd/dedupe/dedupe.go

@@ -32,8 +32,8 @@ By default ` + "`dedupe`" + ` interactively finds files with duplicate
 names and offers to delete all but one or rename them to be
 different. This is known as deduping by name.
 
-Deduping by name is only useful with backends like Google Drive which
-can have duplicate file names. It can be run on wrapping backends
+Deduping by name is only useful with a small group of backends (e.g. Google Drive,
+Opendrive) that can have duplicate file names. It can be run on wrapping backends
 (e.g. crypt) if they wrap a backend which supports duplicate file
 names.
 

cmd/link/link.go

@@ -3,7 +3,6 @@ package link
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
@@ -13,7 +12,7 @@ import (
 )
 
 var (
-	expire = fs.Duration(time.Hour * 24 * 365 * 100)
+	expire = fs.DurationOff
 	unlink = false
 )
 

cmd/listremotes/listremotes.go

@@ -41,7 +41,7 @@ When uses with the -l flag it lists the types too.
 		}
 		for _, remote := range remotes {
 			if listLong {
-				remoteType := config.FileGet(remote, "type", "UNKNOWN")
+				remoteType := config.FileGet(remote, "type")
 				fmt.Printf("%-*s %s\n", maxlen+1, remote+":", remoteType)
 			} else {
 				fmt.Printf("%s:\n", remote)

cmd/mount/dir.go

@@ -181,6 +181,15 @@ func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) (err error) {
 	return nil
 }
 
+// Invalidate a leaf in a directory
+func (d *Dir) invalidateEntry(dirNode fusefs.Node, leaf string) {
+	fs.Debugf(dirNode, "Invalidating %q", leaf)
+	err := d.fsys.server.InvalidateEntry(dirNode, leaf)
+	if err != nil {
+		fs.Debugf(dirNode, "Failed to invalidate %q: %v", leaf, err)
+	}
+}
+
 // Check interface satisfied
 var _ fusefs.NodeRenamer = (*Dir)(nil)
 
@@ -197,6 +206,13 @@ func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs
 		return translateError(err)
 	}
 
+	// Invalidate the new directory entry so it gets re-read (in
+	// the background otherwise we cause a deadlock)
+	//
+	// See https://github.com/rclone/rclone/issues/4977 for why
+	go d.invalidateEntry(newDir, req.NewName)
+	//go d.invalidateEntry(d, req.OldName)
+
 	return nil
 }
 

cmd/mount/fs.go

@@ -20,8 +20,9 @@ import (
 // FS represents the top level filing system
 type FS struct {
 	*vfs.VFS
-	f   fs.Fs
-	opt *mountlib.Options
+	f      fs.Fs
+	opt    *mountlib.Options
+	server *fusefs.Server
 }
 
 // Check interface satisfied

cmd/mount/mount.go

@@ -91,12 +91,12 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error
 	}
 
 	filesys := NewFS(VFS, opt)
-	server := fusefs.New(c, nil)
+	filesys.server = fusefs.New(c, nil)
 
 	// Serve the mount point in the background returning error to errChan
 	errChan := make(chan error, 1)
 	go func() {
-		err := server.Serve(filesys)
+		err := filesys.server.Serve(filesys)
 		closeErr := c.Close()
 		if err == nil {
 			err = closeErr

cmd/mountlib/mount.go

@@ -334,7 +334,7 @@ metadata about files like in UNIX. One case that may arise is that other program
 (incorrectly) interprets this as the file being accessible by everyone. For example
 an SSH client may warn about "unprotected private key file".
 
-WinFsp 2021 (version 1.9, still in beta) introduces a new FUSE option "FileSecurity",
+WinFsp 2021 (version 1.9) introduces a new FUSE option "FileSecurity",
 that allows the complete specification of file security descriptors using
 [SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
 With this you can work around issues such as the mentioned "unprotected private key file"
@@ -342,19 +342,38 @@ by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to t
 
 #### Windows caveats
 
-Note that drives created as Administrator are not visible by other
-accounts (including the account that was elevated as
-Administrator). So if you start a Windows drive from an Administrative
-Command Prompt and then try to access the same drive from Explorer
-(which does not run as Administrator), you will not be able to see the
-new drive.
+Drives created as Administrator are not visible to other accounts,
+not even an account that was elevated to Administrator with the
+User Account Control (UAC) feature. A result of this is that if you mount
+to a drive letter from a Command Prompt run as Administrator, and then try
+to access the same drive from Windows Explorer (which does not run as
+Administrator), you will not be able to see the mounted drive.
 
-The easiest way around this is to start the drive from a normal
-command prompt. It is also possible to start a drive from the SYSTEM
-account (using [the WinFsp.Launcher
-infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
-which creates drives accessible for everyone on the system or
-alternatively using [the nssm service manager](https://nssm.cc/usage).
+If you don't need to access the drive from applications running with
+administrative privileges, the easiest way around this is to always
+create the mount from a non-elevated command prompt.
+
+To make mapped drives available to the user account that created them
+regardless if elevated or not, there is a special Windows setting called
+[linked connections](https://docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry)
+that can be enabled.
+
+It is also possible to make a drive mount available to everyone on the system,
+by running the process creating it as the built-in SYSTEM account.
+There are several ways to do this: One is to use the command-line
+utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec),
+from Microsoft's Sysinternals suite, which has option |-s| to start
+processes as the SYSTEM account. Another alternative is to run the mount
+command from a Windows Scheduled Task, or a Windows Service, configured
+to run as the SYSTEM account. A third alternative is to use the
+[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
+Note that when running rclone as another user, it will not use
+the configuration file from your profile unless you tell it to
+with the [|--config|](https://rclone.org/docs/#config-config-file) option.
+Read more in the [install documentation](https://rclone.org/install/).
+
+Note that mapping to a directory path, instead of a drive letter,
+does not suffer from the same limitations.
 
 ### Limitations
 

cmd/mountlib/rc_test.go

@@ -13,6 +13,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/cmount"
 	_ "github.com/rclone/rclone/cmd/mount"
 	_ "github.com/rclone/rclone/cmd/mount2"
+	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -20,6 +21,7 @@ import (
 
 func TestRc(t *testing.T) {
 	ctx := context.Background()
+	require.NoError(t, configfile.LoadConfig(ctx))
 	mount := rc.Calls.Get("mount/mount")
 	assert.NotNil(t, mount)
 	unmount := rc.Calls.Get("mount/unmount")

cmd/ncdu/ncdu.go

@@ -485,11 +485,15 @@ func (u *UI) removeEntry(pos int) {
 
 // delete the entry at the current position
 func (u *UI) delete() {
+	if u.d == nil || len(u.entries) == 0 {
+		return
+	}
 	ctx := context.Background()
-	dirPos := u.sortPerm[u.dirPosMap[u.path].entry]
-	entry := u.entries[dirPos]
+	cursorPos := u.dirPosMap[u.path]
+	dirPos := u.sortPerm[cursorPos.entry]
+	dirEntry := u.entries[dirPos]
 	u.boxMenu = []string{"cancel", "confirm"}
-	if obj, isFile := entry.(fs.Object); isFile {
+	if obj, isFile := dirEntry.(fs.Object); isFile {
 		u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
 			if o != 1 {
 				return "Aborted!", nil
@@ -499,27 +503,33 @@ func (u *UI) delete() {
 				return "", err
 			}
 			u.removeEntry(dirPos)
+			if cursorPos.entry >= len(u.entries) {
+				u.move(-1) // move back onto a valid entry
+			}
 			return "Successfully deleted file!", nil
 		}
 		u.popupBox([]string{
 			"Delete this file?",
-			u.fsName + entry.String()})
+			u.fsName + dirEntry.String()})
 	} else {
 		u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
 			if o != 1 {
 				return "Aborted!", nil
 			}
-			err := operations.Purge(ctx, f, entry.String())
+			err := operations.Purge(ctx, f, dirEntry.String())
 			if err != nil {
 				return "", err
 			}
 			u.removeEntry(dirPos)
+			if cursorPos.entry >= len(u.entries) {
+				u.move(-1) // move back onto a valid entry
+			}
 			return "Successfully purged folder!", nil
 		}
 		u.popupBox([]string{
 			"Purge this directory?",
 			"ALL files in it will be deleted",
-			u.fsName + entry.String()})
+			u.fsName + dirEntry.String()})
 	}
 }
 

cmd/rcat/rcat.go

@@ -7,12 +7,19 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
 
+var (
+	size = int64(-1)
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.Int64VarP(cmdFlags, &size, "size", "", size, "File size hint to preallocate")
 }
 
 var commandDefinition = &cobra.Command{
@@ -37,6 +44,13 @@ must fit into RAM. The cutoff needs to be small enough to adhere
 the limits of your remote, please see there. Generally speaking,
 setting this cutoff too high will decrease your performance.
 
+Use the |--size| flag to preallocate the file in advance at the remote end
+and actually stream it, even if remote backend doesn't support streaming.
+
+|--size| should be the exact size of the input stream in bytes. If the
+size of the stream is different in length to the |--size| passed in
+then the transfer will likely fail.
+
 Note that the upload can also not be retried because the data is
 not kept around until the upload succeeds. If you need to transfer
 a lot of data, you're better off caching locally and then
@@ -51,7 +65,7 @@ a lot of data, you're better off caching locally and then
 
 		fdst, dstFileName := cmd.NewFsDstFile(args)
 		cmd.Run(false, false, command, func() error {
-			_, err := operations.Rcat(context.Background(), fdst, dstFileName, os.Stdin, time.Now())
+			_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now())
 			return err
 		})
 	},

cmd/rcd/rcd.go

@@ -3,10 +3,13 @@ package rcd
 import (
 	"context"
 	"log"
+	"sync"
 
+	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/spf13/cobra"
 )
 
@@ -48,6 +51,22 @@ See the [rc documentation](/rc/) for more info on the rc flags.
 			log.Fatal("rc server not configured")
 		}
 
+		// Notify stopping on exit
+		var finaliseOnce sync.Once
+		finalise := func() {
+			finaliseOnce.Do(func() {
+				_ = sysdnotify.Stopping()
+			})
+		}
+		fnHandle := atexit.Register(finalise)
+		defer atexit.Unregister(fnHandle)
+
+		// Notify ready to systemd
+		if err := sysdnotify.Ready(); err != nil {
+			log.Fatalf("failed to notify ready to systemd: %v", err)
+		}
+
 		s.Wait()
+		finalise()
 	},
 }

cmd/selfupdate/help.go

@@ -0,0 +1,53 @@
+// +build !noselfupdate
+
+package selfupdate
+
+// Note: "|" will be replaced by backticks in the help string below
+var selfUpdateHelp string = `
+This command downloads the latest release of rclone and replaces
+the currently running binary. The download is verified with a hashsum
+and cryptographically signed signature.
+
+If used without flags (or with implied |--stable| flag), this command
+will install the latest stable release. However, some issues may be fixed
+(or features added) only in the latest beta release. In such cases you should
+run the command with the |--beta| flag, i.e. |rclone selfupdate --beta|.
+You can check in advance what version would be installed by adding the
+|--check| flag, then repeat the command without it when you are satisfied.
+
+Sometimes the rclone team may recommend you a concrete beta or stable
+rclone release to troubleshoot your issue or add a bleeding edge feature.
+The |--version VER| flag, if given, will update to the concrete version
+instead of the latest one. If you omit micro version from |VER| (for
+example |1.53|), the latest matching micro version will be used.
+
+Upon successful update rclone will print a message that contains a previous
+version number. You will need it if you later decide to revert your update
+for some reason. Then you'll have to note the previous version and run the
+following command: |rclone selfupdate [--beta] OLDVER|.
+If the old version contains only dots and digits (for example |v1.54.0|)
+then it's a stable release so you won't need the |--beta| flag. Beta releases
+have an additional information similar to |v1.54.0-beta.5111.06f1c0c61|.
+(if you are a developer and use a locally built rclone, the version number
+will end with |-DEV|, you will have to rebuild it as it obviously can't
+be distributed).
+
+If you previously installed rclone via a package manager, the package may
+include local documentation or configure services. You may wish to update
+with the flag |--package deb| or |--package rpm| (whichever is correct for
+your OS) to update these too. This command with the default |--package zip|
+will update only the rclone executable so the local manual may become
+inaccurate after it.
+
+The |rclone mount| command (https://rclone.org/commands/rclone_mount/) may
+or may not support extended FUSE options depending on the build and OS.
+|selfupdate| will refuse to update if the capability would be discarded.
+
+Note: Windows forbids deletion of a currently running executable so this
+command will rename the old executable to 'rclone.old.exe' upon success.
+
+Please note that this command was not available before rclone version 1.55.
+If it fails for you with the message |unknown command "selfupdate"| then
+you will need to update manually following the install instructions located
+at https://rclone.org/install/
+`

cmd/selfupdate/noselfupdate.go

@@ -0,0 +1,11 @@
+// +build noselfupdate
+
+package selfupdate
+
+import (
+	"github.com/rclone/rclone/lib/buildinfo"
+)
+
+func init() {
+	buildinfo.Tags = append(buildinfo.Tags, "noselfupdate")
+}

cmd/selfupdate/selfupdate.go

@@ -0,0 +1,477 @@
+// +build !noselfupdate
+
+package selfupdate
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/cmount"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/buildinfo"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/spf13/cobra"
+
+	versionCmd "github.com/rclone/rclone/cmd/version"
+)
+
+// Options contains options for the self-update command
+type Options struct {
+	Check   bool
+	Output  string // output path
+	Beta    bool   // mutually exclusive with Stable (false means "stable")
+	Stable  bool   // mutually exclusive with Beta
+	Version string
+	Package string // package format: zip, deb, rpm (empty string means "zip")
+}
+
+// Opt is options set via command line
+var Opt = Options{}
+
+func init() {
+	cmd.Root.AddCommand(cmdSelfUpdate)
+	cmdFlags := cmdSelfUpdate.Flags()
+	flags.BoolVarP(cmdFlags, &Opt.Check, "check", "", Opt.Check, "Check for latest release, do not download.")
+	flags.StringVarP(cmdFlags, &Opt.Output, "output", "", Opt.Output, "Save the downloaded binary at a given path (default: replace running binary)")
+	flags.BoolVarP(cmdFlags, &Opt.Stable, "stable", "", Opt.Stable, "Install stable release (this is the default)")
+	flags.BoolVarP(cmdFlags, &Opt.Beta, "beta", "", Opt.Beta, "Install beta release.")
+	flags.StringVarP(cmdFlags, &Opt.Version, "version", "", Opt.Version, "Install the given rclone version (default: latest)")
+	flags.StringVarP(cmdFlags, &Opt.Package, "package", "", Opt.Package, "Package format: zip|deb|rpm (default: zip)")
+}
+
+var cmdSelfUpdate = &cobra.Command{
+	Use:     "selfupdate",
+	Aliases: []string{"self-update"},
+	Short:   `Update the rclone binary.`,
+	Long:    strings.ReplaceAll(selfUpdateHelp, "|", "`"),
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 0, command, args)
+		if Opt.Package == "" {
+			Opt.Package = "zip"
+		}
+		gotActionFlags := Opt.Stable || Opt.Beta || Opt.Output != "" || Opt.Version != "" || Opt.Package != "zip"
+		if Opt.Check && !gotActionFlags {
+			versionCmd.CheckVersion()
+			return
+		}
+		if Opt.Package != "zip" {
+			if Opt.Package != "deb" && Opt.Package != "rpm" {
+				log.Fatalf("--package should be one of zip|deb|rpm")
+			}
+			if runtime.GOOS != "linux" {
+				log.Fatalf(".deb and .rpm packages are supported only on Linux")
+			} else if os.Geteuid() != 0 && !Opt.Check {
+				log.Fatalf(".deb and .rpm must be installed by root")
+			}
+			if Opt.Output != "" && !Opt.Check {
+				fmt.Println("Warning: --output is ignored with --package deb|rpm")
+			}
+		}
+		if err := InstallUpdate(context.Background(), &Opt); err != nil {
+			log.Fatalf("Error: %v", err)
+		}
+	},
+}
+
+// GetVersion can get the latest release number from the download site
+// or massage a stable release number - prepend semantic "v" prefix
+// or find the latest micro release for a given major.minor release.
+// Note: this will not be applied to beta releases.
+func GetVersion(ctx context.Context, beta bool, version string) (newVersion, siteURL string, err error) {
+	siteURL = "https://downloads.rclone.org"
+	if beta {
+		siteURL = "https://beta.rclone.org"
+	}
+
+	if version == "" {
+		// Request the latest release number from the download site
+		_, newVersion, _, err = versionCmd.GetVersion(siteURL + "/version.txt")
+		return
+	}
+
+	newVersion = version
+	if version[0] != 'v' {
+		newVersion = "v" + version
+	}
+	if beta {
+		return
+	}
+
+	if valid, _ := regexp.MatchString(`^v\d+\.\d+(\.\d+)?$`, newVersion); !valid {
+		return "", siteURL, errors.New("invalid semantic version")
+	}
+
+	// Find the latest stable micro release
+	if strings.Count(newVersion, ".") == 1 {
+		html, err := downloadFile(ctx, siteURL)
+		if err != nil {
+			return "", siteURL, errors.Wrap(err, "failed to get list of releases")
+		}
+		reSubver := fmt.Sprintf(`href="\./%s\.\d+/"`, regexp.QuoteMeta(newVersion))
+		allSubvers := regexp.MustCompile(reSubver).FindAllString(string(html), -1)
+		if allSubvers == nil {
+			return "", siteURL, errors.New("could not find the minor release")
+		}
+		// Use the fact that releases in the index are sorted by date
+		lastSubver := allSubvers[len(allSubvers)-1]
+		newVersion = lastSubver[8 : len(lastSubver)-2]
+	}
+	return
+}
+
+// InstallUpdate performs rclone self-update
+func InstallUpdate(ctx context.Context, opt *Options) error {
+	// Find the latest release number
+	if opt.Stable && opt.Beta {
+		return errors.New("--stable and --beta are mutually exclusive")
+	}
+
+	// The `cmount` tag is added by cmd/cmount/mount.go only if build is static.
+	_, tags := buildinfo.GetLinkingAndTags()
+	if strings.Contains(" "+tags+" ", " cmount ") && !cmount.ProvidedBy(runtime.GOOS) {
+		return errors.New("updating would discard the mount FUSE capability, aborting")
+	}
+
+	newVersion, siteURL, err := GetVersion(ctx, opt.Beta, opt.Version)
+	if err != nil {
+		return errors.Wrap(err, "unable to detect new version")
+	}
+
+	oldVersion := fs.Version
+	if newVersion == oldVersion {
+		fmt.Println("rclone is up to date")
+		return nil
+	}
+
+	// Install .deb/.rpm package if requested by user
+	if opt.Package == "deb" || opt.Package == "rpm" {
+		if opt.Check {
+			fmt.Println("Warning: --package flag is ignored in --check mode")
+		} else {
+			err := installPackage(ctx, opt.Beta, newVersion, siteURL, opt.Package)
+			if err == nil {
+				fmt.Printf("Successfully updated rclone package from version %s to version %s\n", oldVersion, newVersion)
+			}
+			return err
+		}
+	}
+
+	// Get the current executable path
+	executable, err := os.Executable()
+	if err != nil {
+		return errors.Wrap(err, "unable to find executable")
+	}
+
+	targetFile := opt.Output
+	if targetFile == "" {
+		targetFile = executable
+	}
+
+	if opt.Check {
+		fmt.Printf("Without --check this would install rclone version %s at %s\n", newVersion, targetFile)
+		return nil
+	}
+
+	// Make temporary file names and check for possible access errors in advance
+	var newFile string
+	if newFile, err = makeRandomExeName(targetFile, "new"); err != nil {
+		return err
+	}
+	savedFile := ""
+	if runtime.GOOS == "windows" {
+		savedFile = targetFile
+		if strings.HasSuffix(savedFile, ".exe") {
+			savedFile = savedFile[:len(savedFile)-4]
+		}
+		savedFile += ".old.exe"
+	}
+
+	if savedFile == executable || newFile == executable {
+		return fmt.Errorf("%s: a temporary file would overwrite the executable, specify a different --output path", targetFile)
+	}
+
+	if err := verifyAccess(targetFile); err != nil {
+		return err
+	}
+
+	// Download the update as a temporary file
+	err = downloadUpdate(ctx, opt.Beta, newVersion, siteURL, newFile, "zip")
+	if err != nil {
+		return errors.Wrap(err, "failed to update rclone")
+	}
+
+	err = replaceExecutable(targetFile, newFile, savedFile)
+	if err == nil {
+		fmt.Printf("Successfully updated rclone from version %s to version %s\n", oldVersion, newVersion)
+	}
+	return err
+}
+
+func installPackage(ctx context.Context, beta bool, version, siteURL, packageFormat string) error {
+	tempFile, err := ioutil.TempFile("", "rclone.*."+packageFormat)
+	if err != nil {
+		return errors.Wrap(err, "unable to write temporary package")
+	}
+	packageFile := tempFile.Name()
+	_ = tempFile.Close()
+	defer func() {
+		if rmErr := os.Remove(packageFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary package: %v", packageFile, rmErr)
+		}
+	}()
+	if err := downloadUpdate(ctx, beta, version, siteURL, packageFile, packageFormat); err != nil {
+		return err
+	}
+
+	packageCommand := "dpkg"
+	if packageFormat == "rpm" {
+		packageCommand = "rpm"
+	}
+	cmd := exec.Command(packageCommand, "-i", packageFile)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run %s: %v", packageCommand, err)
+	}
+	return nil
+}
+
+func replaceExecutable(targetFile, newFile, savedFile string) error {
+	// Copy permission bits from the old executable
+	// (it was extracted with mode 0755)
+	fileInfo, err := os.Lstat(targetFile)
+	if err == nil {
+		if err = os.Chmod(newFile, fileInfo.Mode()); err != nil {
+			return errors.Wrap(err, "failed to set permission")
+		}
+	}
+
+	if err = os.Remove(targetFile); os.IsNotExist(err) {
+		err = nil
+	}
+
+	if err != nil && savedFile != "" {
+		// Windows forbids removal of a running executable so we rename it.
+		// For starters, rename download as the original file with ".old.exe" appended.
+		var saveErr error
+		if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
+			saveErr = nil
+		}
+		if saveErr == nil {
+			saveErr = os.Rename(targetFile, savedFile)
+		}
+		if saveErr != nil {
+			// The ".old" file cannot be removed or cannot be renamed to.
+			// This usually means that the running executable has a name with ".old".
+			// This can happen in very rare cases, but we ought to handle it.
+			// Try inserting a randomness in the name to mitigate it.
+			fs.Debugf(nil, "%s: cannot replace old file, randomizing name", savedFile)
+
+			savedFile, saveErr = makeRandomExeName(targetFile, "old")
+			if saveErr == nil {
+				if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
+					saveErr = nil
+				}
+			}
+			if saveErr == nil {
+				saveErr = os.Rename(targetFile, savedFile)
+			}
+		}
+		if saveErr == nil {
+			fmt.Printf("The old executable was saved as %s\n", savedFile)
+			err = nil
+		}
+	}
+
+	if err == nil {
+		err = os.Rename(newFile, targetFile)
+	}
+	if err != nil {
+		if rmErr := os.Remove(newFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
+		}
+		return err
+	}
+	return nil
+}
+
+func makeRandomExeName(baseName, extension string) (string, error) {
+	const maxAttempts = 5
+
+	if runtime.GOOS == "windows" {
+		if strings.HasSuffix(baseName, ".exe") {
+			baseName = baseName[:len(baseName)-4]
+		}
+		extension += ".exe"
+	}
+
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		filename := fmt.Sprintf("%s.%s.%s", baseName, random.String(4), extension)
+		if _, err := os.Stat(filename); os.IsNotExist(err) {
+			return filename, nil
+		}
+	}
+
+	return "", fmt.Errorf("cannot find a file name like %s.xxxx.%s", baseName, extension)
+}
+
+func downloadUpdate(ctx context.Context, beta bool, version, siteURL, newFile, packageFormat string) error {
+	osName := runtime.GOOS
+	arch := runtime.GOARCH
+	if arch == "darwin" {
+		arch = "osx"
+	}
+
+	archiveFilename := fmt.Sprintf("rclone-%s-%s-%s.%s", version, osName, arch, packageFormat)
+	archiveURL := fmt.Sprintf("%s/%s/%s", siteURL, version, archiveFilename)
+	archiveBuf, err := downloadFile(ctx, archiveURL)
+	if err != nil {
+		return err
+	}
+	gotHash := sha256.Sum256(archiveBuf)
+	strHash := hex.EncodeToString(gotHash[:])
+	fs.Debugf(nil, "downloaded release archive with hashsum %s from %s", strHash, archiveURL)
+
+	// CI/CD does not provide hashsums for beta releases
+	if !beta {
+		if err := verifyHashsum(ctx, siteURL, version, archiveFilename, gotHash[:]); err != nil {
+			return err
+		}
+	}
+
+	if packageFormat == "deb" || packageFormat == "rpm" {
+		if err := ioutil.WriteFile(newFile, archiveBuf, 0644); err != nil {
+			return errors.Wrap(err, "cannot write temporary ."+packageFormat)
+		}
+		return nil
+	}
+
+	entryName := fmt.Sprintf("rclone-%s-%s-%s/rclone", version, osName, arch)
+	if runtime.GOOS == "windows" {
+		entryName += ".exe"
+	}
+
+	// Extract executable to a temporary file, then replace it by an instant rename
+	err = extractZipToFile(archiveBuf, entryName, newFile)
+	if err != nil {
+		return err
+	}
+	fs.Debugf(nil, "extracted %s to %s", entryName, newFile)
+	return nil
+}
+
+func verifyAccess(file string) error {
+	admin := "root"
+	if runtime.GOOS == "windows" {
+		admin = "Administrator"
+	}
+
+	fileInfo, fileErr := os.Lstat(file)
+
+	if fileErr != nil {
+		dir := filepath.Dir(file)
+		dirInfo, dirErr := os.Lstat(dir)
+		if dirErr != nil {
+			return dirErr
+		}
+		if !dirInfo.Mode().IsDir() {
+			return fmt.Errorf("%s: parent path is not a directory, specify a different path using --output", dir)
+		}
+		if !writable(dir) {
+			return fmt.Errorf("%s: directory is not writable, please run self-update as %s", dir, admin)
+		}
+	}
+
+	if fileErr == nil && !fileInfo.Mode().IsRegular() {
+		return fmt.Errorf("%s: path is not a normal file, specify a different path using --output", file)
+	}
+
+	if fileErr == nil && !writable(file) {
+		return fmt.Errorf("%s: file is not writable, run self-update as %s", file, admin)
+	}
+
+	return nil
+}
+
+func findFileHash(buf []byte, filename string) (hash []byte, err error) {
+	lines := bufio.NewScanner(bytes.NewReader(buf))
+	for lines.Scan() {
+		tokens := strings.Split(lines.Text(), "  ")
+		if len(tokens) == 2 && tokens[1] == filename {
+			if hash, err := hex.DecodeString(tokens[0]); err == nil {
+				return hash, nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("%s: unable to find hash", filename)
+}
+
+func extractZipToFile(buf []byte, entryName, newFile string) error {
+	zipReader, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
+	if err != nil {
+		return err
+	}
+
+	var reader io.ReadCloser
+	for _, entry := range zipReader.File {
+		if entry.Name == entryName {
+			reader, err = entry.Open()
+			break
+		}
+	}
+	if reader == nil || err != nil {
+		return fmt.Errorf("%s: file not found in archive", entryName)
+	}
+	defer func() {
+		_ = reader.Close()
+	}()
+
+	err = os.Remove(newFile)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("%s: unable to create new file: %v", newFile, err)
+	}
+	writer, err := os.OpenFile(newFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(writer, reader)
+	_ = writer.Close()
+	if err != nil {
+		if rmErr := os.Remove(newFile); rmErr != nil {
+			fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
+		}
+	}
+	return err
+}
+
+func downloadFile(ctx context.Context, url string) ([]byte, error) {
+	resp, err := fshttp.NewClient(ctx).Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(resp.Body, &err)
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed with %s downloading %s", resp.Status, url)
+	}
+	return ioutil.ReadAll(resp.Body)
+}