Version v1.54.1

It was my stupid typo: "f.base.NewObject" instead of correct "f.NewObject"
I made it during backport. The commit on master was correct.

.github/ISSUE_TEMPLATE/Bug.md

@@ -5,31 +5,19 @@ about: Report a problem with rclone
 
 <!--
 
-We understand you are having a problem with rclone; we want to help you with that!
+Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
 
-**STOP and READ**
-**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
-Please show the effort you've put in to solving the problem and please be specific.
-People are volunteering their time to help! Low effort posts are not likely to get good answers!
-
-If you think you might have found a bug, try to replicate it with the latest beta (or stable).
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
-
-If you can still replicate it or just got a question then please use the rclone forum:
+If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
 
     https://forum.rclone.org/
 
-for a quick response instead of filing an issue on this repo.
+instead of filing an issue for a quick response.
 
-If nothing else helps, then please fill in the info below which helps us help you.
+If you think you might have found a bug, please can you try to replicate it with the latest beta?
 
-**DO NOT REDACT** any information except passwords/keys/personal info.
-
-You should use 3 backticks to begin and end your paste to make it readable.
-
-Make sure to include a log obtained with '-vv'.
-
-You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
+    https://beta.rclone.org/
+    
+If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
 
 Thank you
 
@@ -37,10 +25,6 @@ The Rclone Developers
 
 -->
 
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
-
 #### What is the problem you are having with rclone?
 
 
@@ -53,7 +37,7 @@ The Rclone Developers
 
 
 
-#### Which cloud storage system are you using? (e.g. Google Drive)
+####  Which cloud storage system are you using? (e.g. Google Drive)
 
 
 
@@ -64,11 +48,3 @@ The Rclone Developers
 #### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
 
 
-
-<!--- Please keep the note below for others who read your bug report. -->
-
-#### How to use GitHub
-
-* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
-* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
-* Subscribe to receive notifications on status change and new comments.

.github/ISSUE_TEMPLATE/Feature.md

@@ -7,16 +7,12 @@ about: Suggest a new feature or enhancement for rclone
 
 Welcome :-)
 
-So you've got an idea to improve rclone? We love that!
-You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
+So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
 
-Probably the latest beta (or stable) release has your feature, so try to update your rclone.
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
+Here is a checklist of things to do:
 
-If it still isn't there, here is a checklist of things to do:
-
-  1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum: https://forum.rclone.org/
+  1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
+  2. Discuss on the forum first: https://forum.rclone.org/
   3. Make a feature request issue (this is the right place!).
   4. Be prepared to get involved making the feature :-)
 
@@ -26,9 +22,6 @@ The Rclone Developers
 
 -->
 
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
 
 #### What is your current rclone version (output from `rclone version`)?
 
@@ -41,11 +34,3 @@ The Rclone Developers
 #### How do you think rclone should be changed to solve that?
 
 
-
-<!--- Please keep the note below for others who read your feature request. -->
-
-#### How to use GitHub
-
-* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
-* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
-* Subscribe to receive notifications on status change and new comments.

.github/workflows/build.yml

@@ -12,52 +12,38 @@ on:
     tags:
       - '*'
   pull_request:
-  workflow_dispatch:
-    inputs:
-      manual:
-        required: true
-        default: true
 
 jobs:
   build:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.13', 'go1.14', 'go1.15']
+        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'go1.12', 'go1.13', 'go1.14', 'go1.16']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.16.x'
+            go: '1.15.x'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
             quicktest: true
             racequicktest: true
-            librclonetest: true
             deploy: true
 
-          - job_name: mac_amd64
+          - job_name: mac
             os: macOS-latest
-            go: '1.16.x'
+            go: '1.15.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
             racequicktest: true
             deploy: true
 
-          - job_name: mac_arm64
-            os: macOS-latest
-            go: '1.16.x'
-            gotags: 'cmount'
-            build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -macos-sdk macosx11.1 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
-            deploy: true
-
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.16.x'
+            go: '1.15.x'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             build_args: '-buildmode exe'
@@ -67,7 +53,7 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.16.x'
+            go: '1.15.x'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -78,11 +64,16 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.16.x'
-            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
+            go: '1.15.x'
+            build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
             compile_all: true
             deploy: true
 
+          - job_name: go1.12
+            os: ubuntu-latest
+            go: '1.12.x'
+            quicktest: true
+
           - job_name: go1.13
             os: ubuntu-latest
             go: '1.13.x'
@@ -94,9 +85,9 @@ jobs:
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.15
+          - job_name: go1.16
             os: ubuntu-latest
-            go: '1.15.x'
+            go: '1.16.x'
             quicktest: true
             racequicktest: true
 
@@ -138,7 +129,7 @@ jobs:
         shell: bash
         run: |
           brew update
-          brew install --cask macfuse
+          brew install --cask osxfuse
         if: matrix.os == 'macOS-latest'
 
       - name: Install Libraries on Windows
@@ -194,14 +185,6 @@ jobs:
           make racequicktest
         if: matrix.racequicktest
 
-      - name: Run librclone tests
-        shell: bash
-        run: |
-          make -C librclone/ctest test
-          make -C librclone/ctest clean
-          librclone/python/test_rclone.py
-        if: matrix.librclonetest
-
       - name: Code quality test
         shell: bash
         run: |
@@ -228,110 +211,50 @@ jobs:
         # Deploy binaries if enabled in config && not a PR && not a fork
         if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
-  android:
-     if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
-     timeout-minutes: 30
-     name: "android-all"
-     runs-on: ubuntu-latest
-     
-     steps:
-       - name: Checkout
-         uses: actions/checkout@v2
-         with:
-           fetch-depth: 0
+  xgo:
+    timeout-minutes: 60
+    name: "xgo cross compile"
+    runs-on: ubuntu-latest
 
-       # Upgrade together with NDK version
-       - name: Set up Go 1.16
-         uses: actions/setup-go@v1
-         with:
-           go-version: 1.16
+    steps:
 
-       # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
-       - name: Force NDK version
-         run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;22.1.7171670" | grep -v = || true
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
 
-       - name: Go module cache
-         uses: actions/cache@v2
-         with:
-           path: ~/go/pkg/mod
-           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-           restore-keys: |
-             ${{ runner.os }}-go-
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo 'GOPATH=${{ runner.workspace }}' >> $GITHUB_ENV
+          echo '${{ runner.workspace }}/bin' >> $GITHUB_PATH
 
-       - name: Set global environment variables
-         shell: bash
-         run: |
-           echo "VERSION=$(make version)" >> $GITHUB_ENV
+      - name: Cross-compile rclone
+        run: |
+          docker pull billziss/xgo-cgofuse
+          GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
+          # xgo \
+          #     -image=billziss/xgo-cgofuse \
+          #     -targets=darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+          #     -tags cmount \
+          #     -dest build \
+          #     .
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=android/*,ios/* \
+              -dest build \
+              .
 
-       - name: build native rclone
-         run: |
-           make
+      - name: Build rclone
+        shell: bash
+        run: |
+          make
 
-       - name: install gomobile
-         run: |
-           go get golang.org/x/mobile/cmd/gobind
-           go get golang.org/x/mobile/cmd/gomobile
-           env PATH=$PATH:~/go/bin gomobile init
-           
-       - name: arm-v7a gomobile build
-         run: env PATH=$PATH:~/go/bin gomobile bind -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile 
-           
-       - name: arm-v7a Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=arm' >> $GITHUB_ENV
-           echo 'GOARM=7' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-       - name: arm-v7a build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
-
-       - name: arm64-v8a Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=arm64' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: arm64-v8a build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
-
-       - name: x86 Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=386' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: x86 build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
-
-       - name: x64 Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=amd64' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: x64 build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
-
-       - name: Upload artifacts
-         run: |
-           make ci_upload
-         env:
-           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-         # Upload artifacts if not a PR && not a fork
-         if: github.head_ref == '' && github.repository == 'rclone/rclone'
+      - name: Upload artifacts
+        run: |
+          make ci_upload
+        env:
+          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
+        # Upload artifacts if not a PR && not a fork
+        if: github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_docker_image.yml

@@ -7,7 +7,6 @@ on:
 
 jobs:
     build:
-        if: github.repository == 'rclone/rclone'
         runs-on: ubuntu-latest
         name: Build image job
         steps:

.github/workflows/build_publish_release_docker_image.yml

@@ -6,7 +6,6 @@ on:
 
 jobs:
     build:
-        if: github.repository == 'rclone/rclone'
         runs-on: ubuntu-latest
         name: Build image job
         steps:
@@ -32,40 +31,3 @@ jobs:
                 publish: true
                 dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
                 dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build and publish docker volume plugin
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v2
-              with:
-                fetch-depth: 0
-            - name: Set plugin parameters
-              shell: bash
-              run: |
-                GITHUB_REF=${{ github.ref }}
-
-                PLUGIN_IMAGE_USER=rclone
-                PLUGIN_IMAGE_NAME=docker-volume-rclone
-                PLUGIN_IMAGE_TAG=${GITHUB_REF#refs/tags/}
-                PLUGIN_IMAGE=${PLUGIN_IMAGE_USER}/${PLUGIN_IMAGE_NAME}:${PLUGIN_IMAGE_TAG}
-                PLUGIN_IMAGE_LATEST=${PLUGIN_IMAGE_USER}/${PLUGIN_IMAGE_NAME}:latest
-
-                echo "PLUGIN_IMAGE_USER=${PLUGIN_IMAGE_USER}" >> $GITHUB_ENV
-                echo "PLUGIN_IMAGE_NAME=${PLUGIN_IMAGE_NAME}" >> $GITHUB_ENV
-                echo "PLUGIN_IMAGE_TAG=${PLUGIN_IMAGE_TAG}" >> $GITHUB_ENV
-                echo "PLUGIN_IMAGE=${PLUGIN_IMAGE}" >> $GITHUB_ENV
-                echo "PLUGIN_IMAGE_LATEST=${PLUGIN_IMAGE_LATEST}" >> $GITHUB_ENV
-            - name: Build image
-              shell: bash
-              run: |
-                make docker-plugin
-            - name: Push image
-              shell: bash
-              run: |
-                docker login -u ${{ secrets.DOCKER_HUB_USER }} -p ${{ secrets.DOCKER_HUB_PASSWORD }}
-                make docker-plugin-push PLUGIN_IMAGE=${PLUGIN_IMAGE}
-                make docker-plugin-push PLUGIN_IMAGE=${PLUGIN_IMAGE_LATEST}

.gitignore

@@ -1,7 +1,6 @@
 *~
 _junk/
 rclone
-rclone.exe
 build
 docs/public
 rclone.iml
@@ -11,6 +10,3 @@ rclone.iml
 *.log
 *.iml
 fuzz-build.zip
-*.orig
-*.rej
-Thumbs.db

CONTRIBUTING.md

@@ -12,162 +12,94 @@ When filing an issue, please include the following information if
 possible as well as a description of the problem.  Make sure you test
 with the [latest beta of rclone](https://beta.rclone.org/):
 
-  * Rclone version (e.g. output from `rclone version`)
-  * Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
+  * Rclone version (e.g. output from `rclone -V`)
+  * Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
   * The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
   * A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
     * if the log contains secrets then edit the file with a text editor first to obscure them
 
-## Submitting a new feature or bug fix ##
+## Submitting a pull request ##
 
 If you find a bug that you'd like to fix, or a new feature that you'd
 like to implement then please submit a pull request via GitHub.
 
-If it is a big feature, then [make an issue](https://github.com/rclone/rclone/issues) first so it can be discussed.
+If it is a big feature then make an issue first so it can be discussed.
 
-To prepare your pull request first press the fork button on [rclone's GitHub
+You'll need a Go environment set up with GOPATH set.  See [the Go
+getting started docs](https://golang.org/doc/install) for more info.
+
+First in your web browser press the fork button on [rclone's GitHub
 page](https://github.com/rclone/rclone).
 
-Then [install Git](https://git-scm.com/downloads) and set your public contribution [name](https://docs.github.com/en/github/getting-started-with-github/setting-your-username-in-git) and [email](https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address#setting-your-commit-email-address-in-git).
+Now in your terminal
 
-Next open your terminal, change directory to your preferred folder and initialise your local rclone project:
-
-    git clone https://github.com/rclone/rclone.git
-    cd rclone
+    go get -u github.com/rclone/rclone
+    cd $GOPATH/src/github.com/rclone/rclone
     git remote rename origin upstream
-      # if you have SSH keys setup in your GitHub account:
     git remote add origin git@github.com:YOURUSER/rclone.git
-      # otherwise:
-    git remote add origin https://github.com/YOURUSER/rclone.git
 
-Note that most of the terminal commands in the rest of this guide must be executed from the rclone folder created above.
-
-Now [install Go](https://golang.org/doc/install) and verify your installation:
-
-    go version
-
-Great, you can now compile and execute your own version of rclone:
-
-    go build                              
-    ./rclone version
-
-Finally make a branch to add your new feature
+Make a branch to add your new feature
 
     git checkout -b my-new-feature
 
 And get hacking.
 
-You may like one of the [popular editors/IDE's for Go](https://github.com/golang/go/wiki/IDEsAndTextEditorPlugins) and a quick view on the rclone [code organisation](#code-organisation).
+When ready - run the unit tests for the code you changed
 
-When ready - test the affected functionality and run the unit tests for the code you changed
-
-    cd folder/with/changed/files
     go test -v
 
 Note that you may need to make a test remote, e.g. `TestSwift` for some
 of the unit tests.
 
-This is typically enough if you made a simple bug fix, otherwise please read the rclone [testing](#testing) section too.
+Note the top level Makefile targets
+
+  * make check
+  * make test
+
+Both of these will be run by Travis when you make a pull request but
+you can do this yourself locally too.  These require some extra go
+packages which you can install with
+
+  * make build_dep
 
 Make sure you
 
-  * Add [unit tests](#testing) for a new feature.
   * Add [documentation](#writing-documentation) for a new feature.
-  * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
+  * Follow the [commit message guidelines](#commit-messages).
+  * Add [unit tests](#testing) for a new feature
+  * squash commits down to one per feature
+  * rebase to master with `git rebase master`
 
-When you are done with that push your changes to Github:
+When you are done with that
 
-    git push -u origin my-new-feature
+    git push origin my-new-feature
 
-and open the GitHub website to [create your pull
+Go to the GitHub website and click [Create pull
 request](https://help.github.com/articles/creating-a-pull-request/).
 
-Your changes will then get reviewed and you might get asked to fix some stuff. If so, then make the changes in the same branch, commit and push your updates to GitHub.
+You patch will get reviewed and you might get asked to fix some stuff.
 
-You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
+If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
+```
+git log # See how many commits you want to squash
+git reset --soft HEAD~2 # This squashes the 2 latest commits together.
+git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
+git commit # Add a new commit message.
+git push --force # Push the squashed commit to your GitHub repo.
+# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also recommends wizardzines.com
+```
 
-## Using Git and Github ##
-
-### Committing your changes ###
-
-Follow the guideline for [commit messages](#commit-messages) and then:
-
-    git checkout my-new-feature      # To switch to your branch
-    git status                       # To see the new and changed files
-    git add FILENAME                 # To select FILENAME for the commit
-    git status                       # To verify the changes to be committed
-    git commit                       # To do the commit
-    git log                          # To verify the commit. Use q to quit the log
-
-You can modify the message or changes in the latest commit using:
-
-    git commit --amend
-
-If you amend to commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-### Replacing your previously pushed commits ###
-
-Note that you are about to rewrite the GitHub history of your branch. It is good practice to involve your collaborators before modifying commits that have been pushed to GitHub.
-
-Your previously pushed commits are replaced by:
-
-    git push --force origin my-new-feature 
-
-### Basing your changes on the latest master ###
-
-To base your changes on the latest version of the [rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
-
-    git checkout master
-    git fetch upstream
-    git merge --ff-only
-    git push origin --follow-tags    # optional update of your fork in GitHub
-    git checkout my-new-feature
-    git rebase master
-
-If you rebase commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-### Squashing your commits ###
-
-To combine your commits into one commit:
-
-    git log                          # To count the commits to squash, e.g. the last 2
-    git reset --soft HEAD~2          # To undo the 2 latest commits
-    git status                       # To check everything is as expected
-
-If everything is fine, then make the new combined commit:
-
-    git commit                       # To commit the undone commits as one
-
-otherwise, you may roll back using:
-
-    git reflog                       # To check that HEAD{1} is your previous state
-    git reset --soft 'HEAD@{1}'      # To roll back to your previous state
-
-If you squash commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-Tip: You may like to use `git rebase -i master` if you are experienced or have a more complex situation.
-
-### GitHub Continuous Integration ###
+## CI for your fork ##
 
 rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
 ## Testing ##
 
-### Quick testing ###
-
 rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
 
     go test -v ./...
-
-You can also use `make`, if supported by your platform
-
-    make quicktest
-
-The quicktest is [automatically run by GitHub](#github-continuous-integration) when you push your branch to GitHub.
-
-### Backend testing ###
-
+    
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
 storage systems by mocking all their interfaces, rclone unit tests can
@@ -183,8 +115,8 @@ are skipped if `TestDrive:` isn't defined.
     cd backend/drive
     go test -v
 
-You can then run the integration tests which test all of rclone's
-operations.  Normally these get run against the local file system,
+You can then run the integration tests which tests all of rclone's
+operations.  Normally these get run against the local filing system,
 but they can be run against any of the remotes.
 
     cd fs/sync
@@ -195,25 +127,18 @@ but they can be run against any of the remotes.
     go test -v -remote TestDrive:
 
 If you want to use the integration test framework to run these tests
-altogether with an HTML report and test retries then from the
+all together with an HTML report and test retries then from the
 project root:
 
     go install github.com/rclone/rclone/fstest/test_all
     test_all -backend drive
 
-### Full integration testing ###
-
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
 
-    make check
     make test
 
-The commands may require some extra go packages which you can install with
-
-    make build_dep
-
-The full integration tests are run daily on the integration test server. You can
+This command is run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##
@@ -228,7 +153,6 @@ with modules beneath.
   * cmd - the rclone commands
     * all - import this to load all the commands
     * ...commands
-  * cmdtest - end-to-end tests of commands, flags, environment variables,...
   * docs - the documentation and website
     * content - adjust these docs only - everything else is autogenerated
       * command - these are auto generated - edit the corresponding .go file
@@ -278,7 +202,7 @@ for the flag help, the remainder is shown to the user in `rclone
 config` and is added to the docs with `make backenddocs`.
 
 The only documentation you need to edit are the `docs/content/*.md`
-files.  The `MANUAL.*`, `rclone.1`, web site, etc. are all auto generated
+files.  The MANUAL.*, rclone.1, web site, etc. are all auto generated
 from those during the release process.  See the `make doc` and `make
 website` targets in the Makefile if you are interested in how.  You
 don't need to run these when adding a feature.
@@ -341,7 +265,7 @@ rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-rclone can be built with modules outside of the `GOPATH`.
+rclone can be built with modules outside of the GOPATH
 
 To add a dependency `github.com/ncw/new_dependency` see the
 instructions below.  These will fetch the dependency and add it to
@@ -409,8 +333,8 @@ Getting going
   * Try to implement as many optional methods as possible as it makes the remote more usable.
   * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
     * `rclone purge -v TestRemote:rclone-info`
-    * `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    * `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
+    * `rclone info --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    * `go run cmd/info/internal/build_csv/main.go -o remote.csv remote.json`
     * open `remote.csv` in a spreadsheet and examine
 
 Unit tests
@@ -476,7 +400,7 @@ Usage
    - If this variable doesn't exist, plugin support is disabled.
    - Plugins must be compiled against the exact version of rclone to work.
      (The rclone used during building the plugin must be the same as the source of rclone)
-
+     
 Building
 
 To turn your existing additions into a Go plugin, move them to an external repository

Dockerfile

@@ -16,8 +16,6 @@ RUN apk --no-cache add ca-certificates fuse tzdata && \
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 
-RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
-
 ENTRYPOINT [ "rclone" ]
 
 WORKDIR /data

Makefile

@@ -119,7 +119,7 @@ doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 rclone.1:	MANUAL.md
 	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
@@ -187,10 +187,10 @@ upload_github:
 	./bin/upload-github $(TAG)
 
 cross:	doc
-	go run bin/cross-compile.go -release current $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -release current $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 beta:
-	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
@@ -198,7 +198,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -compile-only $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 ci_upload:
 	sudo chown -R $$USER build
@@ -256,36 +256,3 @@ startstable:
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
-
-# docker volume plugin
-PLUGIN_IMAGE_USER ?= rclone
-PLUGIN_IMAGE_TAG ?= latest
-PLUGIN_IMAGE_NAME ?= docker-volume-rclone
-PLUGIN_IMAGE ?= $(PLUGIN_IMAGE_USER)/$(PLUGIN_IMAGE_NAME):$(PLUGIN_IMAGE_TAG)
-
-PLUGIN_BASE_IMAGE := rclone/rclone:latest
-PLUGIN_BUILD_DIR := ./build/docker-plugin
-PLUGIN_CONTRIB_DIR := ./cmd/serve/docker/contrib/plugin
-PLUGIN_CONFIG := $(PLUGIN_CONTRIB_DIR)/config.json
-PLUGIN_DOCKERFILE := $(PLUGIN_CONTRIB_DIR)/Dockerfile
-PLUGIN_CONTAINER := docker-volume-rclone-dev-$(shell date +'%Y%m%d-%H%M%S')
-
-docker-plugin: docker-plugin-rootfs docker-plugin-create
-
-docker-plugin-image: rclone
-	docker build --no-cache --pull --build-arg BASE_IMAGE=${PLUGIN_BASE_IMAGE} -t ${PLUGIN_IMAGE} -f ${PLUGIN_DOCKERFILE} .
-
-docker-plugin-rootfs: docker-plugin-image
-	mkdir -p ${PLUGIN_BUILD_DIR}/rootfs
-	docker create --name ${PLUGIN_CONTAINER} ${PLUGIN_IMAGE}
-	docker export ${PLUGIN_CONTAINER} | tar -x -C ${PLUGIN_BUILD_DIR}/rootfs
-	docker rm -vf ${PLUGIN_CONTAINER}
-	cp ${PLUGIN_CONFIG} ${PLUGIN_BUILD_DIR}/config.json
-
-docker-plugin-create:
-	docker plugin rm -f ${PLUGIN_IMAGE} 2>/dev/null || true
-	docker plugin create ${PLUGIN_IMAGE} ${PLUGIN_BUILD_DIR}
-
-docker-plugin-push: docker-plugin-create
-	docker plugin push ${PLUGIN_IMAGE}
-	docker plugin rm ${PLUGIN_IMAGE}

README.md

@@ -62,7 +62,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
-  * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
@@ -88,6 +87,7 @@ Please see [the full list of all storage providers and their features](https://r
   * Optional large file chunking ([Chunker](https://rclone.org/chunker/))
   * Optional transparent compression ([Compress](https://rclone.org/compress/))
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
+  * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
   * Multi-threaded downloads to local disk
   * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna

RELEASE.md

@@ -4,7 +4,7 @@ This file describes how to make the various kinds of releases
 
 ## Extra required software for making a release
 
-  * [gh the github cli](https://github.com/cli/cli) for uploading packages
+  * [github-release](https://github.com/aktau/github-release) for uploading packages
   * pandoc for making the html and man pages
 
 ## Making a release
@@ -76,24 +76,6 @@ Now
 The rclone docker image should autobuild on via GitHub actions.  If it doesn't
 or needs to be updated then rebuild like this.
 
-See: https://github.com/ilteoood/docker_buildx/issues/19
-See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
-
-```
-git co v1.54.1
-docker pull golang
-export DOCKER_CLI_EXPERIMENTAL=enabled
-docker buildx create --name actions_builder --use
-docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
-echo "Supported platforms: $SUPPORTED_PLATFORMS"
-docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
-docker buildx stop actions_builder
-```
-
-### Old build for linux/amd64 only
-
 ```
 docker pull golang
 docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .

VERSION

@@ -1 +1 @@
-v1.56.2
+v1.54.1

backend/alias/alias_internal_test.go

@@ -11,7 +11,6 @@ import (
 	_ "github.com/rclone/rclone/backend/local" // pull in test backend
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,7 +19,7 @@ var (
 )
 
 func prepare(t *testing.T, root string) {
-	configfile.Install()
+	config.LoadConfig(context.Background())
 
 	// Configure the remote
 	config.FileSet(remoteName, "type", "alias")

backend/all/all.go

@@ -41,7 +41,6 @@ import (
 	_ "github.com/rclone/rclone/backend/swift"
 	_ "github.com/rclone/rclone/backend/tardigrade"
 	_ "github.com/rclone/rclone/backend/union"
-	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"
 	_ "github.com/rclone/rclone/backend/yandex"
 	_ "github.com/rclone/rclone/backend/zoho"

backend/amazonclouddrive/amazonclouddrive.go

@@ -16,6 +16,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"path"
 	"strings"
@@ -69,10 +70,11 @@ func init() {
 		Prefix:      "acd",
 		Description: "Amazon Drive",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: acdConfig,
-			})
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+			err := oauthutil.Config(ctx, "amazon cloud drive", name, m, acdConfig, nil)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "checkpoint",
@@ -81,16 +83,16 @@ func init() {
 			Advanced: true,
 		}, {
 			Name: "upload_wait_per_gb",
-			Help: `Additional time per GiB to wait after a failed complete upload to see if it appears.
+			Help: `Additional time per GB to wait after a failed complete upload to see if it appears.
 
 Sometimes Amazon Drive gives an error when a file has been fully
 uploaded but the file appears anyway after a little while.  This
-happens sometimes for files over 1 GiB in size and nearly every time for
-files bigger than 10 GiB. This parameter controls the time rclone waits
+happens sometimes for files over 1GB in size and nearly every time for
+files bigger than 10GB. This parameter controls the time rclone waits
 for the file to appear.
 
-The default value for this parameter is 3 minutes per GiB, so by
-default it will wait 3 minutes for every GiB uploaded to see if the
+The default value for this parameter is 3 minutes per GB, so by
+default it will wait 3 minutes for every GB uploaded to see if the
 file appears.
 
 You can disable this feature by setting it to 0. This may cause
@@ -110,7 +112,7 @@ in this situation.`,
 
 Files this size or more will be downloaded via their "tempLink". This
 is to work around a problem with Amazon Drive which blocks downloads
-of files bigger than about 10 GiB. The default for this is 9 GiB which
+of files bigger than about 10GB.  The default for this is 9GB which
 shouldn't need to be changed.
 
 To download files above this threshold, rclone requests a "tempLink"
@@ -203,10 +205,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	if resp != nil {
 		if resp.StatusCode == 401 {
 			f.tokenRenewer.Invalidate()
@@ -281,7 +280,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.getRootInfo(ctx)
+		_, err := f.getRootInfo()
 		return err
 	})
 
@@ -289,14 +288,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		_, resp, err = f.c.Account.GetEndpoints()
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get endpoints")
 	}
 
 	// Get rootID
-	rootInfo, err := f.getRootInfo(ctx)
+	rootInfo, err := f.getRootInfo()
 	if err != nil || rootInfo.Id == nil {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
@@ -338,11 +337,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // getRootInfo gets the root folder info
-func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) {
+func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		rootInfo, resp, err = f.c.Nodes.GetRoot()
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	return rootInfo, err
 }
@@ -381,7 +380,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	var subFolder *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -408,7 +407,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var info *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -429,7 +428,7 @@ type listAllFn func(*acd.Node) bool
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	query := "parents:" + dirID
 	if directoriesOnly {
 		query += " AND kind:" + folderKind
@@ -450,7 +449,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, title string, directorie
 		var resp *http.Response
 		err = f.pacer.CallNoRetry(func() (bool, error) {
 			nodes, resp, err = f.c.Nodes.GetNodes(&opts)
-			return f.shouldRetry(ctx, resp, err)
+			return f.shouldRetry(resp, err)
 		})
 		if err != nil {
 			return false, err
@@ -509,7 +508,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var iErr error
 	for tries := 1; tries <= maxTries; tries++ {
 		entries = nil
-		_, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool {
 			remote := path.Join(dir, *node.Name)
 			switch *node.Kind {
 			case folderKind:
@@ -668,7 +667,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		if ok {
 			return false, nil
 		}
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -709,7 +708,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
+	err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
 	if err != nil {
 		return nil, err
 	}
@@ -804,7 +803,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var jsonStr string
 	err = srcFs.pacer.Call(func() (bool, error) {
 		jsonStr, err = srcInfo.GetMetadata()
-		return srcFs.shouldRetry(ctx, nil, err)
+		return srcFs.shouldRetry(nil, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error: error reading src metadata: %v", err)
@@ -816,7 +815,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return err
 	}
 
-	err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
+	err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
 	if err != nil {
 		return err
 	}
@@ -841,7 +840,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if check {
 		// check directory is empty
 		empty := true
-		_, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool {
+		_, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool {
 			switch *node.Kind {
 			case folderKind:
 				empty = false
@@ -866,7 +865,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = node.Trash()
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -988,7 +987,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	var info *acd.File
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if err == acd.ErrorNodeNotFound {
@@ -1045,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		} else {
 			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	return in, err
 }
@@ -1068,7 +1067,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if ok {
 			return false, nil
 		}
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1078,70 +1077,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove a node
-func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error {
+func (f *Fs) removeNode(info *acd.Node) error {
 	var resp *http.Response
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = info.Trash()
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	return err
 }
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.removeNode(ctx, o.info)
+	return o.fs.removeNode(o.info)
 }
 
 // Restore a node
-func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) {
+func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Restore()
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	return newInfo, err
 }
 
 // Changes name of given node
-func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) {
+func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	return newInfo, err
 }
 
 // Replaces one parent with another, effectively moving the file. Leaves other
 // parents untouched. ReplaceParent cannot be used when the file is trashed.
-func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error {
+func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.ReplaceParent(oldParentID, newParentID)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 }
 
 // Adds one additional parent to object.
-func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error {
+func (f *Fs) addParent(info *acd.Node, newParentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.AddParent(newParentID)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 }
 
 // Remove given parent from object, leaving the other possible
 // parents untouched. Object can end up having no parents.
-func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error {
+func (f *Fs) removeParent(info *acd.Node, parentID string) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := info.RemoveParent(parentID)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 }
 
 // moveNode moves the node given from the srcLeaf,srcDirectoryID to
 // the dstLeaf,dstDirectoryID
-func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
+func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
 	// fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID)
 	cantMove := fs.ErrorCantMove
 	if useDirErrorMsgs {
@@ -1155,7 +1154,7 @@ func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string,
 
 	if srcLeaf != dstLeaf {
 		// fs.Debugf(name, "renaming")
-		_, err = f.renameNode(ctx, srcInfo, dstLeaf)
+		_, err = f.renameNode(srcInfo, dstLeaf)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path rename failed: %v", err)
 			goto OnConflict
@@ -1163,7 +1162,7 @@ func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string,
 	}
 	if srcDirectoryID != dstDirectoryID {
 		// fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID)
-		err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID)
+		err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID)
 		if err != nil {
 			fs.Debugf(name, "Move: quick path parent replace failed: %v", err)
 			return err
@@ -1176,13 +1175,13 @@ OnConflict:
 	fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.")
 
 	// fs.Debugf(name, "Trashing file")
-	err = f.removeNode(ctx, srcInfo)
+	err = f.removeNode(srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: remove node failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Renaming file")
-	_, err = f.renameNode(ctx, srcInfo, dstLeaf)
+	_, err = f.renameNode(srcInfo, dstLeaf)
 	if err != nil {
 		fs.Debugf(name, "Move: rename node failed: %v", err)
 		return err
@@ -1190,19 +1189,19 @@ OnConflict:
 	// note: replacing parent is forbidden by API, modifying them individually is
 	// okay though
 	// fs.Debugf(name, "Adding target parent")
-	err = f.addParent(ctx, srcInfo, dstDirectoryID)
+	err = f.addParent(srcInfo, dstDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: addParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "removing original parent")
-	err = f.removeParent(ctx, srcInfo, srcDirectoryID)
+	err = f.removeParent(srcInfo, srcDirectoryID)
 	if err != nil {
 		fs.Debugf(name, "Move: removeParent failed: %v", err)
 		return err
 	}
 	// fs.Debugf(name, "Restoring")
-	_, err = f.restoreNode(ctx, srcInfo)
+	_, err = f.restoreNode(srcInfo)
 	if err != nil {
 		fs.Debugf(name, "Move: restoreNode node failed: %v", err)
 		return err

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -1,6 +1,5 @@
 // Test AmazonCloudDrive filesystem interface
 
-//go:build acd
 // +build acd
 
 package amazonclouddrive_test

backend/azureblob/azureblob.go

@@ -1,6 +1,5 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-//go:build !plan9 && !solaris && !js && go1.14
 // +build !plan9,!solaris,!js,go1.14
 
 package azureblob
@@ -48,8 +47,8 @@ const (
 	timeFormatIn          = time.RFC3339
 	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
 	storageDefaultBaseURL = "blob.core.windows.net"
-	defaultChunkSize      = 4 * fs.Mebi
-	maxChunkSize          = 100 * fs.Mebi
+	defaultChunkSize      = 4 * fs.MebiByte
+	maxChunkSize          = 100 * fs.MebiByte
 	uploadConcurrency     = 4
 	defaultAccessTier     = azblob.AccessTierNone
 	maxTryTimeout         = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
@@ -81,12 +80,13 @@ func init() {
 
 Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
 
-    $ az ad sp create-for-rbac --name "<name>" \
+    $ az sp create-for-rbac --name "<name>" \
       --role "Storage Blob Data Owner" \
       --scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
       > azure-principal.json
 
-See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+See [Use Azure CLI to assign an Azure role for access to blob and queue data](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli)
+for more details.
 `,
 		}, {
 			Name: "key",
@@ -129,11 +129,11 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Advanced: true,
 		}, {
 			Name:     "upload_cutoff",
-			Help:     "Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)",
+			Help:     "Cutoff for switching to chunked upload (<= 256MB). (Deprecated)",
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
-			Help: `Upload chunk size (<= 100 MiB).
+			Help: `Upload chunk size (<= 100MB).
 
 Note that this is stored in memory and there may be up to
 "--transfers" chunks stored at once in memory.`,
@@ -217,23 +217,6 @@ This option controls how often unused buffers will be removed from the pool.`,
 				encoder.EncodeDel |
 				encoder.EncodeBackSlash |
 				encoder.EncodeRightPeriod),
-		}, {
-			Name:    "public_access",
-			Help:    "Public access level of a container: blob, container.",
-			Default: string(azblob.PublicAccessNone),
-			Examples: []fs.OptionExample{
-				{
-					Value: string(azblob.PublicAccessNone),
-					Help:  "The container and its blobs can be accessed only with an authorized request. It's a default value",
-				}, {
-					Value: string(azblob.PublicAccessBlob),
-					Help:  "Blob data within this container can be read via anonymous request.",
-				}, {
-					Value: string(azblob.PublicAccessContainer),
-					Help:  "Allow full public read access for container and blob data.",
-				},
-			},
-			Advanced: true,
 		}},
 	})
 }
@@ -258,7 +241,6 @@ type Options struct {
 	MemoryPoolFlushTime  fs.Duration          `config:"memory_pool_flush_time"`
 	MemoryPoolUseMmap    bool                 `config:"memory_pool_use_mmap"`
 	Enc                  encoder.MultiEncoder `config:"encoding"`
-	PublicAccess         string               `config:"public_access"`
 }
 
 // Fs represents a remote azure server
@@ -280,7 +262,6 @@ type Fs struct {
 	imdsPacer     *fs.Pacer                       // Same but for IMDS
 	uploadToken   *pacer.TokenDispenser           // control concurrency
 	pool          *pool.Pool                      // memory pool
-	publicAccess  azblob.PublicAccessType         // Container Public Access Level
 }
 
 // Object describes an azure object
@@ -354,19 +335,6 @@ func validateAccessTier(tier string) bool {
 	}
 }
 
-// validatePublicAccess checks if azureblob supports use supplied public access level
-func validatePublicAccess(publicAccess string) bool {
-	switch publicAccess {
-	case string(azblob.PublicAccessNone),
-		string(azblob.PublicAccessBlob),
-		string(azblob.PublicAccessContainer):
-		// valid cases
-		return true
-	default:
-		return false
-	}
-}
-
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	401, // Unauthorized (e.g. "Token has expired")
@@ -379,10 +347,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetry(err error) (bool, error) {
 	// FIXME interpret special errors - more to do here
 	if storageErr, ok := err.(azblob.StorageError); ok {
 		switch storageErr.ServiceCode() {
@@ -404,7 +369,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 }
 
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	const minChunkSize = fs.SizeSuffixBase
+	const minChunkSize = fs.Byte
 	if cs < minChunkSize {
 		return errors.Errorf("%s is less than %s", cs, minChunkSize)
 	}
@@ -534,11 +499,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
 	}
 
-	if !validatePublicAccess((opt.PublicAccess)) {
-		return nil, errors.Errorf("Azure Blob: Supported public access level are %s and %s",
-			string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer))
-	}
-
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
 		name:        name,
@@ -557,7 +517,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			opt.MemoryPoolUseMmap,
 		),
 	}
-	f.publicAccess = azblob.PublicAccessType(opt.PublicAccess)
 	f.imdsPacer.SetRetries(5) // per IMDS documentation
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -619,7 +578,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// Retry as specified by the documentation:
 			// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
 			token, err = GetMSIToken(ctx, userMSI)
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 
 		if err != nil {
@@ -635,7 +594,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			var refreshedToken adal.Token
 			err := f.imdsPacer.Call(func() (bool, error) {
 				refreshedToken, err = GetMSIToken(ctx, userMSI)
-				return f.shouldRetry(ctx, err)
+				return f.shouldRetry(err)
 			})
 			if err != nil {
 				// Failed to refresh.
@@ -844,7 +803,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 
 		if err != nil {
@@ -1070,7 +1029,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return err
@@ -1122,7 +1081,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 		}
 		// now try to create the container
 		return f.pacer.Call(func() (bool, error) {
-			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, f.publicAccess)
+			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
 			if err != nil {
 				if storageErr, ok := err.(azblob.StorageError); ok {
 					switch storageErr.ServiceCode() {
@@ -1139,7 +1098,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 					}
 				}
 			}
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 	}, nil)
 }
@@ -1177,10 +1136,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error {
 					return false, fs.ErrorDirNotFound
 				}
 
-				return f.shouldRetry(ctx, err)
+				return f.shouldRetry(err)
 			}
 
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 	})
 }
@@ -1253,7 +1212,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	err = f.pacer.Call(func() (bool, error) {
 		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1414,7 +1373,7 @@ func (o *Object) readMetaData() (err error) {
 	var blobProperties *azblob.BlobGetPropertiesResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
@@ -1449,7 +1408,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	blob := o.getBlobReference()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -1492,7 +1451,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var downloadResponse *azblob.DownloadResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open for download")
@@ -1633,7 +1592,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Stream contents of the reader object to the given blob URL
 		blockBlobURL := blob.ToBlockBlobURL()
 		_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -1661,7 +1620,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	ac := azblob.BlobAccessConditions{}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.Delete(ctx, snapShotOptions, ac)
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 }
 
@@ -1690,7 +1649,7 @@ func (o *Object) SetTier(tier string) error {
 	ctx := context.Background()
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 
 	if err != nil {

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !solaris && !js && go1.14
 // +build !plan9,!solaris,!js,go1.14
 
 package azureblob

backend/azureblob/azureblob_test.go

@@ -1,6 +1,5 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js && go1.14
 // +build !plan9,!solaris,!js,go1.14
 
 package azureblob

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js || !go1.14
 // +build plan9 solaris js !go1.14
 
 package azureblob

backend/azureblob/imds.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !solaris && !js && go1.14
 // +build !plan9,!solaris,!js,go1.14
 
 package azureblob

backend/azureblob/imds_test.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !solaris && !js && go1.14
 // +build !plan9,!solaris,!js,go1.14
 
 package azureblob

backend/b2/api/types.go

@@ -2,11 +2,12 @@ package api
 
 import (
 	"fmt"
+	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/version"
 )
 
 // Error describes a B2 error response
@@ -62,17 +63,16 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-// HasVersion returns true if it looks like the passed filename has a timestamp on it.
-//
-// Note that the passed filename's timestamp may still be invalid even if this
-// function returns true.
-func HasVersion(remote string) bool {
-	return version.Match(remote)
-}
+const versionFormat = "-v2006-01-02-150405.000"
 
 // AddVersion adds the timestamp as a version string into the filename passed in.
 func (t Timestamp) AddVersion(remote string) string {
-	return version.Add(remote, time.Time(t))
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	s := time.Time(t).Format(versionFormat)
+	// Replace the '.' with a '-'
+	s = strings.Replace(s, ".", "-", -1)
+	return base + s + ext
 }
 
 // RemoveVersion removes the timestamp from a filename as a version string.
@@ -80,9 +80,24 @@ func (t Timestamp) AddVersion(remote string) string {
 // It returns the new file name and a timestamp, or the old filename
 // and a zero timestamp.
 func RemoveVersion(remote string) (t Timestamp, newRemote string) {
-	time, newRemote := version.Remove(remote)
-	t = Timestamp(time)
-	return
+	newRemote = remote
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	if len(base) < len(versionFormat) {
+		return
+	}
+	versionStart := len(base) - len(versionFormat)
+	// Check it ends in -xxx
+	if base[len(base)-4] != '-' {
+		return
+	}
+	// Replace with .xxx for parsing
+	base = base[:len(base)-4] + "." + base[len(base)-3:]
+	newT, err := time.Parse(versionFormat, base[versionStart:])
+	if err != nil {
+		return
+	}
+	return Timestamp(newT), base[:versionStart] + ext
 }
 
 // IsZero returns true if the timestamp is uninitialized

backend/b2/api/types_test.go

@@ -13,6 +13,7 @@ import (
 var (
 	emptyT api.Timestamp
 	t0     = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
+	t0r    = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
 	t1     = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
 )
 
@@ -35,6 +36,40 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
 	assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
 }
 
+func TestTimestampAddVersion(t *testing.T) {
+	for _, test := range []struct {
+		t        api.Timestamp
+		in       string
+		expected string
+	}{
+		{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
+		{t1, "potato", "potato-v2001-02-03-040506-123"},
+		{t1, "", "-v2001-02-03-040506-123"},
+	} {
+		actual := test.t.AddVersion(test.in)
+		assert.Equal(t, test.expected, actual, test.in)
+	}
+}
+
+func TestTimestampRemoveVersion(t *testing.T) {
+	for _, test := range []struct {
+		in             string
+		expectedT      api.Timestamp
+		expectedRemote string
+	}{
+		{"potato.txt", emptyT, "potato.txt"},
+		{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
+		{"potato-v2001-02-03-040506-123", t1, "potato"},
+		{"-v2001-02-03-040506-123", t1, ""},
+		{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
+		{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
+	} {
+		actualT, actualRemote := api.RemoveVersion(test.in)
+		assert.Equal(t, test.expectedT, actualT, test.in)
+		assert.Equal(t, test.expectedRemote, actualRemote, test.in)
+	}
+}
+
 func TestTimestampIsZero(t *testing.T) {
 	assert.True(t, emptyT.IsZero())
 	assert.False(t, t0.IsZero())

backend/b2/b2.go

@@ -54,10 +54,10 @@ const (
 	decayConstant       = 1 // bigger for slower decay, exponential
 	maxParts            = 10000
 	maxVersions         = 100 // maximum number of versions we search in --b2-versions mode
-	minChunkSize        = 5 * fs.Mebi
-	defaultChunkSize    = 96 * fs.Mebi
-	defaultUploadCutoff = 200 * fs.Mebi
-	largeFileCopyCutoff = 4 * fs.Gibi              // 5E9 is the max
+	minChunkSize        = 5 * fs.MebiByte
+	defaultChunkSize    = 96 * fs.MebiByte
+	defaultUploadCutoff = 200 * fs.MebiByte
+	largeFileCopyCutoff = 4 * fs.GibiByte          // 5E9 is the max
 	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
 	memoryPoolUseMmap   = false
 )
@@ -116,7 +116,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 
 Files above this size will be uploaded in chunks of "--b2-chunk-size".
 
-This value should be set no larger than 4.657 GiB (== 5 GB).`,
+This value should be set no larger than 4.657GiB (== 5GB).`,
 			Default:  defaultUploadCutoff,
 			Advanced: true,
 		}, {
@@ -126,7 +126,7 @@ This value should be set no larger than 4.657 GiB (== 5 GB).`,
 Any files larger than this that need to be server-side copied will be
 copied in chunks of this size.
 
-The minimum is 0 and the maximum is 4.6 GiB.`,
+The minimum is 0 and the maximum is 4.6GB.`,
 			Default:  largeFileCopyCutoff,
 			Advanced: true,
 		}, {
@@ -305,10 +305,7 @@ var retryErrorCodes = []int{
 
 // shouldRetryNoAuth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 	// For 429 or 503 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.
@@ -339,7 +336,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 		}
 		return true, err
 	}
-	return f.shouldRetryNoReauth(ctx, resp, err)
+	return f.shouldRetryNoReauth(resp, err)
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -507,7 +504,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
-		return f.shouldRetryNoReauth(ctx, resp, err)
+		return f.shouldRetryNoReauth(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to authenticate")
@@ -1353,7 +1350,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
 	}
 	var request = api.GetDownloadAuthorizationRequest{
 		BucketID:               bucketID,
-		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.rootDirectory, remote)),
+		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
 		ValidDurationInSeconds: validDurationInSeconds,
 	}
 	var response api.GetDownloadAuthorizationResponse
@@ -1744,13 +1741,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		ContentType:     resp.Header.Get("Content-Type"),
 		Info:            Info,
 	}
-	// When reading files from B2 via cloudflare using
-	// --b2-download-url cloudflare strips the Content-Length
-	// headers (presumably so it can inject stuff) so use the old
-	// length read from the listing.
-	if info.Size < 0 {
-		info.Size = o.size
-	}
 	return resp, info, nil
 }
 

backend/b2/upload.go

@@ -230,14 +230,14 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 		//
 		// The number of bytes in the file being uploaded. Note that
 		// this header is required; you cannot leave it out and just
-		// use chunked encoding. The minimum size of every part but
-		// the last one is 100 MB (100,000,000 bytes)
+		// use chunked encoding.  The minimum size of every part but
+		// the last one is 100MB.
 		//
 		// X-Bz-Content-Sha1
 		//
 		// The SHA1 checksum of the this part of the file. B2 will
 		// check this when the part is uploaded, to make sure that the
-		// data arrived correctly. The same SHA1 checksum must be
+		// data arrived correctly.  The same SHA1 checksum must be
 		// passed to b2_finish_large_file.
 		opts := rest.Opts{
 			Method:  "POST",

backend/box/api/types.go

@@ -36,13 +36,13 @@ func (t *Time) UnmarshalJSON(data []byte) error {
 
 // Error is returned from box when things go wrong
 type Error struct {
-	Type        string          `json:"type"`
-	Status      int             `json:"status"`
-	Code        string          `json:"code"`
-	ContextInfo json.RawMessage `json:"context_info"`
-	HelpURL     string          `json:"help_url"`
-	Message     string          `json:"message"`
-	RequestID   string          `json:"request_id"`
+	Type        string `json:"type"`
+	Status      int    `json:"status"`
+	Code        string `json:"code"`
+	ContextInfo json.RawMessage
+	HelpURL     string `json:"help_url"`
+	Message     string `json:"message"`
+	RequestID   string `json:"request_id"`
 }
 
 // Error returns a string for the error and satisfies the error interface
@@ -132,38 +132,6 @@ type UploadFile struct {
 	ContentModifiedAt Time   `json:"content_modified_at"`
 }
 
-// PreUploadCheck is the request for upload preflight check
-type PreUploadCheck struct {
-	Name   string `json:"name"`
-	Parent Parent `json:"parent"`
-	Size   *int64 `json:"size,omitempty"`
-}
-
-// PreUploadCheckResponse is the response from upload preflight check
-// if successful
-type PreUploadCheckResponse struct {
-	UploadToken string `json:"upload_token"`
-	UploadURL   string `json:"upload_url"`
-}
-
-// PreUploadCheckConflict is returned in the ContextInfo error field
-// from PreUploadCheck when the error code is "item_name_in_use"
-type PreUploadCheckConflict struct {
-	Conflicts struct {
-		Type        string `json:"type"`
-		ID          string `json:"id"`
-		FileVersion struct {
-			Type string `json:"type"`
-			ID   string `json:"id"`
-			Sha1 string `json:"sha1"`
-		} `json:"file_version"`
-		SequenceID string `json:"sequence_id"`
-		Etag       string `json:"etag"`
-		Sha1       string `json:"sha1"`
-		Name       string `json:"name"`
-	} `json:"conflicts"`
-}
-
 // UpdateFileModTime is used in Update File Info
 type UpdateFileModTime struct {
 	ContentModifiedAt Time `json:"content_modified_at"`

backend/box/box.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -83,7 +84,7 @@ func init() {
 		Name:        "box",
 		Description: "Box",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
 			jsonFile, ok := m.Get("box_config_file")
 			boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 			boxAccessToken, boxAccessTokenOk := m.Get("access_token")
@@ -92,15 +93,15 @@ func init() {
 			if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
 				err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
 				if err != nil {
-					return nil, errors.Wrap(err, "failed to configure token with jwt authentication")
+					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
 				}
 				// Else, if not using an access token, use oauth2
 			} else if boxAccessToken == "" || !boxAccessTokenOk {
-				return oauthutil.ConfigOut("", &oauthutil.Options{
-					OAuth2Config: oauthConfig,
-				})
+				err = oauthutil.Config(ctx, "box", name, m, oauthConfig, nil)
+				if err != nil {
+					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
+				}
 			}
-			return nil, nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "root_folder_id",
@@ -125,7 +126,7 @@ func init() {
 			}},
 		}, {
 			Name:     "upload_cutoff",
-			Help:     "Cutoff for switching to multipart upload (>= 50 MiB).",
+			Help:     "Cutoff for switching to multipart upload (>= 50MB).",
 			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
@@ -156,15 +157,15 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	jsonFile = env.ShellExpand(jsonFile)
 	boxConfig, err := getBoxConfig(jsonFile)
 	if err != nil {
-		return errors.Wrap(err, "get box config")
+		log.Fatalf("Failed to configure token: %v", err)
 	}
 	privateKey, err := getDecryptedPrivateKey(boxConfig)
 	if err != nil {
-		return errors.Wrap(err, "get decrypted private key")
+		log.Fatalf("Failed to configure token: %v", err)
 	}
 	claims, err := getClaims(boxConfig, boxSubType)
 	if err != nil {
-		return errors.Wrap(err, "get claims")
+		log.Fatalf("Failed to configure token: %v", err)
 	}
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
@@ -316,13 +317,10 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	authRetry := false
 
-	if resp != nil && resp.StatusCode == 401 && strings.Contains(resp.Header.Get("Www-Authenticate"), "expired_token") {
+	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
 		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
@@ -550,7 +548,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -587,7 +585,7 @@ OUTER:
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -685,80 +683,22 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 	return o, leaf, directoryID, nil
 }
 
-// preUploadCheck checks to see if a file can be uploaded
-//
-// It returns "", nil if the file is good to go
-// It returns "ID", nil if the file must be updated
-func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
-	check := api.PreUploadCheck{
-		Name: f.opt.Enc.FromStandardName(leaf),
-		Parent: api.Parent{
-			ID: directoryID,
-		},
-	}
-	if size >= 0 {
-		check.Size = &size
-	}
-	opts := rest.Opts{
-		Method: "OPTIONS",
-		Path:   "/files/content/",
-	}
-	var result api.PreUploadCheckResponse
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(ctx, &opts, &check, &result)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "item_name_in_use" {
-			var conflict api.PreUploadCheckConflict
-			err = json.Unmarshal(apiErr.ContextInfo, &conflict)
-			if err != nil {
-				return "", errors.Wrap(err, "pre-upload check: JSON decode failed")
-			}
-			if conflict.Conflicts.Type != api.ItemTypeFile {
-				return "", errors.Wrap(err, "pre-upload check: can't overwrite non file with file")
-			}
-			return conflict.Conflicts.ID, nil
-		}
-		return "", errors.Wrap(err, "pre-upload check")
-	}
-	return "", nil
-}
-
 // Put the object
 //
 // Copy the reader in to the new object which is returned
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	// If directory doesn't exist, file doesn't exist so can upload
-	remote := src.Remote()
-	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		if err == fs.ErrorDirNotFound {
-			return f.PutUnchecked(ctx, in, src, options...)
-		}
+	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
+	switch err {
+	case nil:
+		return existingObj, existingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		// Not found so create it
+		return f.PutUnchecked(ctx, in, src)
+	default:
 		return nil, err
 	}
-
-	// Preflight check the upload, which returns the ID if the
-	// object already exists
-	ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
-	if err != nil {
-		return nil, err
-	}
-	if ID == "" {
-		return f.PutUnchecked(ctx, in, src, options...)
-	}
-
-	// If object exists then create a skeleton one with just id
-	o := &Object{
-		fs:     f,
-		remote: remote,
-		id:     ID,
-	}
-	return o, o.Update(ctx, in, src, options...)
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
@@ -800,7 +740,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
@@ -827,7 +767,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -899,7 +839,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -937,7 +877,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -955,7 +895,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read user info")
@@ -1068,7 +1008,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return info.SharedLink.URL, err
 }
@@ -1086,7 +1026,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 	}
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
@@ -1108,7 +1048,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list trash")
@@ -1242,7 +1182,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return info, err
 }
@@ -1275,7 +1215,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1285,7 +1225,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // upload does a single non-multipart upload
 //
-// This is recommended for less than 50 MiB of content
+// This is recommended for less than 50 MB of content
 func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
 	upload := api.UploadFile{
 		Name:              o.fs.opt.Enc.FromStandardName(leaf),
@@ -1315,7 +1255,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err

backend/box/upload.go

@@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return
 }
@@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -109,10 +109,10 @@ outer:
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
-				return shouldRetry(ctx, resp, err)
+				return shouldRetry(resp, err)
 			}
 			body, err = rest.ReadBody(resp)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		delay := defaultDelay
 		var why string
@@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return err
 }

backend/cache/cache.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache
@@ -99,14 +98,14 @@ changed, any downloaded chunks will be invalid and cache-chunk-path
 will need to be cleared or unexpected EOF errors will occur.`,
 			Default: DefCacheChunkSize,
 			Examples: []fs.OptionExample{{
-				Value: "1M",
-				Help:  "1 MiB",
+				Value: "1m",
+				Help:  "1MB",
 			}, {
 				Value: "5M",
-				Help:  "5 MiB",
+				Help:  "5 MB",
 			}, {
 				Value: "10M",
-				Help:  "10 MiB",
+				Help:  "10 MB",
 			}},
 		}, {
 			Name: "info_age",
@@ -133,13 +132,13 @@ oldest chunks until it goes under this value.`,
 			Default: DefCacheTotalChunkSize,
 			Examples: []fs.OptionExample{{
 				Value: "500M",
-				Help:  "500 MiB",
+				Help:  "500 MB",
 			}, {
 				Value: "1G",
-				Help:  "1 GiB",
+				Help:  "1 GB",
 			}, {
 				Value: "10G",
-				Help:  "10 GiB",
+				Help:  "10 GB",
 			}},
 		}, {
 			Name:     "db_path",
@@ -340,14 +339,8 @@ func parseRootPath(path string) (string, error) {
 	return strings.Trim(path, "/"), nil
 }
 
-var warnDeprecated sync.Once
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
-	warnDeprecated.Do(func() {
-		fs.Logf(nil, "WARNING: Cache backend is deprecated and may be removed in future. Please use VFS instead.")
-	})
-
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)

backend/cache/cache_internal_test.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
+// +build !plan9,!js
+// +build !race
 
 package cache_test
 
@@ -836,7 +836,7 @@ func newRun() *run {
 	if uploadDir == "" {
 		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
 		if err != nil {
-			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
+			log.Fatalf("Failed to create temp dir: %v", err)
 		}
 	} else {
 		r.tmpUploadDir = uploadDir
@@ -892,7 +892,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.FileGet(remote, "type")
+		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -903,14 +903,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.FileGet(remote, "remote")
+			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.FileGet(remoteWrapping, "type")
+			remoteType := config.FileGet(remoteWrapping, "type", "")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil

backend/cache/cache_test.go

@@ -1,7 +1,7 @@
 // Test Cache filesystem interface
 
-//go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
+// +build !plan9,!js
+// +build !race
 
 package cache_test
 

backend/cache/cache_unsupported.go

@@ -1,7 +1,6 @@
 // Build for cache for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || js
 // +build plan9 js
 
 package cache

backend/cache/cache_upload_test.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
+// +build !plan9,!js
+// +build !race
 
 package cache_test
 

backend/cache/directory.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/cache/handle.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/cache/object.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/cache/plex.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/cache/storage_memory.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/cache/storage_persistent.go

@@ -1,4 +1,3 @@
-//go:build !plan9 && !js
 // +build !plan9,!js
 
 package cache

backend/chunker/chunker.go

@@ -47,8 +47,7 @@ import (
 // The following types of chunks are supported:
 // data and control, active and temporary.
 // Chunk type is identified by matching chunk file name
-// based on the chunk name format configured by user and transaction
-// style being used.
+// based on the chunk name format configured by user.
 //
 // Both data and control chunks can be either temporary (aka hidden)
 // or active (non-temporary aka normal aka permanent).
@@ -64,12 +63,6 @@ import (
 // which is transparently converted to the new format. In its maximum
 // length of 13 decimals it makes a 7-digit base-36 number.
 //
-// When transactions is set to the norename style, data chunks will
-// keep their temporary chunk names (with the transacion identifier
-// suffix). To distinguish them from temporary chunks, the txn field
-// of the metadata file is set to match the transaction identifier of
-// the data chunks.
-//
 // Chunker can tell data chunks from control chunks by the characters
 // located in the "hash placeholder" position of configured format.
 // Data chunks have decimal digits there.
@@ -108,7 +101,7 @@ const maxMetadataSize = 1023
 const maxMetadataSizeWritten = 255
 
 // Current/highest supported metadata format.
-const metadataVersion = 2
+const metadataVersion = 1
 
 // optimizeFirstChunk enables the following optimization in the Put:
 // If a single chunk is expected, put the first chunk using the
@@ -155,7 +148,7 @@ Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
 		}, {
 			Name:     "chunk_size",
 			Advanced: false,
-			Default:  fs.SizeSuffix(2147483648), // 2 GiB
+			Default:  fs.SizeSuffix(2147483648), // 2GB
 			Help:     `Files larger than chunk size will be split in chunks.`,
 		}, {
 			Name:     "name_format",
@@ -231,31 +224,6 @@ It has the following fields: ver, size, nchunks, md5, sha1.`,
 					Help:  "Warn user, skip incomplete file and proceed.",
 				},
 			},
-		}, {
-			Name:     "transactions",
-			Advanced: true,
-			Default:  "rename",
-			Help:     `Choose how chunker should handle temporary files during transactions.`,
-			Hide:     fs.OptionHideCommandLine,
-			Examples: []fs.OptionExample{
-				{
-					Value: "rename",
-					Help:  "Rename temporary files after a successful transaction.",
-				}, {
-					Value: "norename",
-					Help: `Leave temporary file names and write transaction ID to metadata file.
-Metadata is required for no rename transactions (meta format cannot be "none").
-If you are using norename transactions you should be careful not to downgrade Rclone
-as older versions of Rclone don't support this transaction style and will misinterpret
-files manipulated by norename transactions.
-This method is EXPERIMENTAL, don't use on production systems.`,
-				}, {
-					Value: "auto",
-					Help: `Rename or norename will be used depending on capabilities of the backend.
-If meta format is set to "none", rename transactions will always be used.
-This method is EXPERIMENTAL, don't use on production systems.`,
-				},
-			},
 		}},
 	})
 }
@@ -277,10 +245,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		return nil, errors.New("can't point remote at itself - check the value of the remote setting")
 	}
 
-	baseName, basePath, err := fspath.SplitFs(remote)
+	baseName, basePath, err := fspath.Parse(remote)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
+	if baseName != "" {
+		baseName += ":"
+	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(basePath, rpath)
 	baseFs, err := cache.Get(ctx, baseName+remotePath)
@@ -300,7 +271,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
-	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType, opt.Transactions); err != nil {
+	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType); err != nil {
 		return nil, err
 	}
 
@@ -338,14 +309,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Remote       string        `config:"remote"`
-	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
-	NameFormat   string        `config:"name_format"`
-	StartFrom    int           `config:"start_from"`
-	MetaFormat   string        `config:"meta_format"`
-	HashType     string        `config:"hash_type"`
-	FailHard     bool          `config:"fail_hard"`
-	Transactions string        `config:"transactions"`
+	Remote     string        `config:"remote"`
+	ChunkSize  fs.SizeSuffix `config:"chunk_size"`
+	NameFormat string        `config:"name_format"`
+	StartFrom  int           `config:"start_from"`
+	MetaFormat string        `config:"meta_format"`
+	HashType   string        `config:"hash_type"`
+	FailHard   bool          `config:"fail_hard"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -367,13 +337,12 @@ type Fs struct {
 	opt          Options        // copy of Options
 	features     *fs.Features   // optional features
 	dirSort      bool           // reserved for future, ignored
-	useNoRename  bool           // can be set with the transactions option
 }
 
 // configure sets up chunker for given name format, meta format and hash type.
 // It also seeds the source of random transaction identifiers.
 // configure must be called only from NewFs or by unit tests.
-func (f *Fs) configure(nameFormat, metaFormat, hashType, transactionMode string) error {
+func (f *Fs) configure(nameFormat, metaFormat, hashType string) error {
 	if err := f.setChunkNameFormat(nameFormat); err != nil {
 		return errors.Wrapf(err, "invalid name format '%s'", nameFormat)
 	}
@@ -383,9 +352,6 @@ func (f *Fs) configure(nameFormat, metaFormat, hashType, transactionMode string)
 	if err := f.setHashType(hashType); err != nil {
 		return err
 	}
-	if err := f.setTransactionMode(transactionMode); err != nil {
-		return err
-	}
 
 	randomSeed := time.Now().UnixNano()
 	f.xactIDRand = rand.New(rand.NewSource(randomSeed))
@@ -445,27 +411,6 @@ func (f *Fs) setHashType(hashType string) error {
 	return nil
 }
 
-func (f *Fs) setTransactionMode(transactionMode string) error {
-	switch transactionMode {
-	case "rename":
-		f.useNoRename = false
-	case "norename":
-		if !f.useMeta {
-			return errors.New("incompatible transaction options")
-		}
-		f.useNoRename = true
-	case "auto":
-		f.useNoRename = !f.CanQuickRename()
-		if f.useNoRename && !f.useMeta {
-			f.useNoRename = false
-			return errors.New("using norename transactions requires metadata")
-		}
-	default:
-		return fmt.Errorf("unsupported transaction mode '%s'", transactionMode)
-	}
-	return nil
-}
-
 // setChunkNameFormat converts pattern based chunk name format
 // into Printf format and Regular expressions for data and
 // control chunks.
@@ -748,7 +693,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 	byRemote := make(map[string]*Object)
 	badEntry := make(map[string]bool)
 	isSubdir := make(map[string]bool)
-	txnByRemote := map[string]string{}
 
 	var tempEntries fs.DirEntries
 	for _, dirOrObject := range sortedEntries {
@@ -761,18 +705,12 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 				object := f.newObject("", entry, nil)
 				byRemote[remote] = object
 				tempEntries = append(tempEntries, object)
-				if f.useNoRename {
-					txnByRemote[remote], err = object.readXactID(ctx)
-					if err != nil {
-						return nil, err
-					}
-				}
 				break
 			}
 			// this is some kind of chunk
 			// metobject should have been created above if present
+			isSpecial := xactID != "" || ctrlType != ""
 			mainObject := byRemote[mainRemote]
-			isSpecial := xactID != txnByRemote[mainRemote] || ctrlType != ""
 			if mainObject == nil && f.useMeta && !isSpecial {
 				fs.Debugf(f, "skip orphan data chunk %q", remote)
 				break
@@ -871,11 +809,10 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 	}
 
 	var (
-		o             *Object
-		baseObj       fs.Object
-		currentXactID string
-		err           error
-		sameMain      bool
+		o        *Object
+		baseObj  fs.Object
+		err      error
+		sameMain bool
 	)
 
 	if f.useMeta {
@@ -919,14 +856,7 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		return nil, errors.Wrap(err, "can't detect composite file")
 	}
 
-	if f.useNoRename {
-		currentXactID, err = o.readXactID(ctx)
-		if err != nil {
-			return nil, err
-		}
-	}
 	caseInsensitive := f.features.CaseInsensitive
-
 	for _, dirOrObject := range entries {
 		entry, ok := dirOrObject.(fs.Object)
 		if !ok {
@@ -948,7 +878,7 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		if !sameMain {
 			continue // skip alien chunks
 		}
-		if ctrlType != "" || xactID != currentXactID {
+		if ctrlType != "" || xactID != "" {
 			if f.useMeta {
 				// temporary/control chunk calls for lazy metadata read
 				o.unsure = true
@@ -1063,57 +993,12 @@ func (o *Object) readMetadata(ctx context.Context) error {
 		}
 		o.md5 = metaInfo.md5
 		o.sha1 = metaInfo.sha1
-		o.xactID = metaInfo.xactID
 	}
 
 	o.isFull = true // cache results
-	o.xIDCached = true
 	return nil
 }
 
-// readXactID returns the transaction ID stored in the passed metadata object
-func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
-	// if xactID has already been read and cahced return it now
-	if o.xIDCached {
-		return o.xactID, nil
-	}
-	// Avoid reading metadata for backends that don't use xactID to identify permanent chunks
-	if !o.f.useNoRename {
-		return "", errors.New("readXactID requires norename transactions")
-	}
-	if o.main == nil {
-		return "", errors.New("readXactID requires valid metaobject")
-	}
-	if o.main.Size() > maxMetadataSize {
-		return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
-	}
-	reader, err := o.main.Open(ctx)
-	if err != nil {
-		return "", err
-	}
-	data, err := ioutil.ReadAll(reader)
-	_ = reader.Close() // ensure file handle is freed on windows
-	if err != nil {
-		return "", err
-	}
-
-	switch o.f.opt.MetaFormat {
-	case "simplejson":
-		if data != nil && len(data) > maxMetadataSizeWritten {
-			return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
-		}
-		var metadata metaSimpleJSON
-		err = json.Unmarshal(data, &metadata)
-		if err != nil {
-			return "", nil // this was likely not a metadata object, return empty xactID but don't throw error
-		}
-		xactID = metadata.XactID
-	}
-	o.xactID = xactID
-	o.xIDCached = true
-	return xactID, nil
-}
-
 // put implements Put, PutStream, PutUnchecked, Update
 func (f *Fs) put(
 	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
@@ -1266,17 +1151,14 @@ func (f *Fs) put(
 	// If previous object was chunked, remove its chunks
 	f.removeOldChunks(ctx, baseRemote)
 
-	if !f.useNoRename {
-		// The transaction suffix will be removed for backends with quick rename operations
-		for chunkNo, chunk := range c.chunks {
-			chunkRemote := f.makeChunkName(baseRemote, chunkNo, "", "")
-			chunkMoved, errMove := f.baseMove(ctx, chunk, chunkRemote, delFailed)
-			if errMove != nil {
-				return nil, errMove
-			}
-			c.chunks[chunkNo] = chunkMoved
+	// Rename data chunks from temporary to final names
+	for chunkNo, chunk := range c.chunks {
+		chunkRemote := f.makeChunkName(baseRemote, chunkNo, "", "")
+		chunkMoved, errMove := f.baseMove(ctx, chunk, chunkRemote, delFailed)
+		if errMove != nil {
+			return nil, errMove
 		}
-		xactID = ""
+		c.chunks[chunkNo] = chunkMoved
 	}
 
 	if !f.useMeta {
@@ -1296,7 +1178,7 @@ func (f *Fs) put(
 	switch f.opt.MetaFormat {
 	case "simplejson":
 		c.updateHashes()
-		metadata, err = marshalSimpleJSON(ctx, sizeTotal, len(c.chunks), c.md5, c.sha1, xactID)
+		metadata, err = marshalSimpleJSON(ctx, sizeTotal, len(c.chunks), c.md5, c.sha1)
 	}
 	if err == nil {
 		metaInfo := f.wrapInfo(src, baseRemote, int64(len(metadata)))
@@ -1308,7 +1190,6 @@ func (f *Fs) put(
 
 	o := f.newObject("", metaObject, c.chunks)
 	o.size = sizeTotal
-	o.xactID = xactID
 	return o, nil
 }
 
@@ -1448,7 +1329,7 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
 		c.accountBytes(size)
 		return nil
 	}
-	const bufLen = 1048576 // 1 MiB
+	const bufLen = 1048576 // 1MB
 	buf := make([]byte, bufLen)
 	for size > 0 {
 		n := size
@@ -1712,7 +1593,7 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 	var metadata []byte
 	switch f.opt.MetaFormat {
 	case "simplejson":
-		metadata, err = marshalSimpleJSON(ctx, newObj.size, len(newChunks), md5, sha1, o.xactID)
+		metadata, err = marshalSimpleJSON(ctx, newObj.size, len(newChunks), md5, sha1)
 		if err == nil {
 			metaInfo := f.wrapInfo(metaObject, "", int64(len(metadata)))
 			err = newObj.main.Update(ctx, bytes.NewReader(metadata), metaInfo)
@@ -1928,13 +1809,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		//fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
 		if entryType == fs.EntryObject {
 			mainPath, _, _, xactID := f.parseChunkName(path)
-			metaXactID := ""
-			if f.useNoRename {
-				metaObject, _ := f.base.NewObject(ctx, mainPath)
-				dummyObject := f.newObject("", metaObject, nil)
-				metaXactID, _ = dummyObject.readXactID(ctx)
-			}
-			if mainPath != "" && xactID == metaXactID {
+			if mainPath != "" && xactID == "" {
 				path = mainPath
 			}
 		}
@@ -1955,17 +1830,15 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // Object represents a composite file wrapping one or more data chunks
 type Object struct {
-	remote    string
-	main      fs.Object   // meta object if file is composite, or wrapped non-chunked file, nil if meta format is 'none'
-	chunks    []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
-	size      int64       // cached total size of chunks in a composite file or -1 for non-chunked files
-	isFull    bool        // true if metadata has been read
-	xIDCached bool        // true if xactID has been read
-	unsure    bool        // true if need to read metadata to detect object type
-	xactID    string      // transaction ID for "norename" or empty string for "renamed" chunks
-	md5       string
-	sha1      string
-	f         *Fs
+	remote string
+	main   fs.Object   // meta object if file is composite, or wrapped non-chunked file, nil if meta format is 'none'
+	chunks []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
+	size   int64       // cached total size of chunks in a composite file or -1 for non-chunked files
+	isFull bool        // true if metadata has been read
+	unsure bool        // true if need to read metadata to detect object type
+	md5    string
+	sha1   string
+	f      *Fs
 }
 
 func (o *Object) addChunk(chunk fs.Object, chunkNo int) error {
@@ -2293,7 +2166,6 @@ type ObjectInfo struct {
 	src     fs.ObjectInfo
 	fs      *Fs
 	nChunks int    // number of data chunks
-	xactID  string // transaction ID for "norename" or empty string for "renamed" chunks
 	size    int64  // overrides source size by the total size of data chunks
 	remote  string // overrides remote name
 	md5     string // overrides MD5 checksum
@@ -2392,9 +2264,8 @@ type metaSimpleJSON struct {
 	Size     *int64 `json:"size"`    // total size of data chunks
 	ChunkNum *int   `json:"nchunks"` // number of data chunks
 	// optional extra fields
-	MD5    string `json:"md5,omitempty"`
-	SHA1   string `json:"sha1,omitempty"`
-	XactID string `json:"txn,omitempty"` // transaction ID for norename transactions
+	MD5  string `json:"md5,omitempty"`
+	SHA1 string `json:"sha1,omitempty"`
 }
 
 // marshalSimpleJSON
@@ -2404,20 +2275,16 @@ type metaSimpleJSON struct {
 // - if file contents can be mistaken as meta object
 // - if consistent hashing is On but wrapped remote can't provide given hash
 //
-func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) {
+func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1 string) ([]byte, error) {
 	version := metadataVersion
-	if xactID == "" && version == 2 {
-		version = 1
-	}
 	metadata := metaSimpleJSON{
 		// required core fields
 		Version:  &version,
 		Size:     &size,
 		ChunkNum: &nChunks,
 		// optional extra fields
-		MD5:    md5,
-		SHA1:   sha1,
-		XactID: xactID,
+		MD5:  md5,
+		SHA1: sha1,
 	}
 	data, err := json.Marshal(&metadata)
 	if err == nil && data != nil && len(data) >= maxMetadataSizeWritten {
@@ -2495,7 +2362,6 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	info.nChunks = *metadata.ChunkNum
 	info.md5 = metadata.MD5
 	info.sha1 = metadata.SHA1
-	info.xactID = metadata.XactID
 	return info, true, nil
 }
 
@@ -2528,11 +2394,6 @@ func (f *Fs) Precision() time.Duration {
 	return f.base.Precision()
 }
 
-// CanQuickRename returns true if the Fs supports a quick rename operation
-func (f *Fs) CanQuickRename() bool {
-	return f.base.Features().Move != nil
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/chunker/chunker_internal_test.go

@@ -33,7 +33,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
 		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
 			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
 			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
-			Size:    int64(kilobytes) * int64(fs.Kibi),
+			Size:    int64(kilobytes) * int64(fs.KibiByte),
 		})
 	})
 }
@@ -468,15 +468,9 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 		return obj
 	}
 	billyObj := newFile("billy")
-	billyTxn := billyObj.(*Object).xactID
-	if f.useNoRename {
-		require.True(t, billyTxn != "")
-	} else {
-		require.True(t, billyTxn == "")
-	}
 
 	billyChunkName := func(chunkNo int) string {
-		return f.makeChunkName(billyObj.Remote(), chunkNo, "", billyTxn)
+		return f.makeChunkName(billyObj.Remote(), chunkNo, "", "")
 	}
 
 	err := f.Mkdir(ctx, billyChunkName(1))
@@ -493,8 +487,6 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	// accessing chunks in strict mode is prohibited
 	f.opt.FailHard = true
 	billyChunk4Name := billyChunkName(4)
-	_, err = f.base.NewObject(ctx, billyChunk4Name)
-	require.NoError(t, err)
 	_, err = f.NewObject(ctx, billyChunk4Name)
 	assertOverlapError(err)
 
@@ -528,8 +520,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 
 	// recreate billy in case it was anyhow corrupted
 	willyObj := newFile("willy")
-	willyTxn := willyObj.(*Object).xactID
-	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", willyTxn)
+	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", "")
 	f.opt.FailHard = false
 	willyChunk, err := f.NewObject(ctx, willyChunkName)
 	f.opt.FailHard = true
@@ -570,20 +561,17 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	contents := random.String(100)
 
-	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
-		filename = path.Join(dir, name)
+	newFile := func(f fs.Fs, name string) (fs.Object, string) {
+		filename := path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
-		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
-			txnID = chunkObj.xactID
-		}
-		return
+		return obj, filename
 	}
 
 	f.opt.FailHard = false
-	file, fileName, fileTxn := newFile(f, "wreaker")
-	wreak, _, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", fileTxn))
+	file, fileName := newFile(f, "wreaker")
+	wreak, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", ""))
 
 	f.opt.FailHard = false
 	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
@@ -662,7 +650,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		}
 	}
 
-	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "", "")
+	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "")
 	require.NoError(t, err)
 	todaysMeta := string(metaData)
 	runSubtest(todaysMeta, "today")
@@ -676,7 +664,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 	runSubtest(futureMeta, "future")
 }
 
-// Test that chunker refuses to change on objects with future/unknown metadata
+// test that chunker refuses to change on objects with future/unknowm metadata
 func testFutureProof(t *testing.T, f *Fs) {
 	if f.opt.MetaFormat == "none" {
 		t.Skip("this test requires metadata support")
@@ -750,100 +738,6 @@ func testFutureProof(t *testing.T, f *Fs) {
 	}
 }
 
-// The newer method of doing transactions without renaming should still be able to correctly process chunks that were created with renaming
-// If you attempt to do the inverse, however, the data chunks will be ignored causing commands to perform incorrectly
-func testBackwardsCompatibility(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't do norename transactions without metadata")
-	}
-	const dir = "backcomp"
-	ctx := context.Background()
-	saveOpt := f.opt
-	saveUseNoRename := f.useNoRename
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-		f.useNoRename = saveUseNoRename
-	}()
-	f.opt.ChunkSize = fs.SizeSuffix(10)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	contents := random.String(250)
-	newFile := func(f fs.Fs, name string) (fs.Object, string) {
-		filename := path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		return obj, filename
-	}
-
-	f.opt.FailHard = false
-	f.useNoRename = false
-	file, fileName := newFile(f, "renamefile")
-
-	f.opt.FailHard = false
-	item := fstest.NewItem(fileName, contents, modTime)
-
-	var items []fstest.Item
-	items = append(items, item)
-
-	f.useNoRename = true
-	fstest.CheckListingWithRoot(t, f, dir, items, nil, f.Precision())
-	_, err := f.NewObject(ctx, fileName)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = true
-	_, err = f.List(ctx, dir)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = false
-	_ = file.Remove(ctx)
-}
-
-func testChunkerServerSideMove(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't test norename transactions without metadata")
-	}
-
-	ctx := context.Background()
-	const dir = "servermovetest"
-	subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), dir)
-
-	subFs1, err := fs.NewFs(ctx, subRemote+"/subdir1")
-	assert.NoError(t, err)
-	fs1, isChunkerFs := subFs1.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs1.useNoRename = false
-	fs1.opt.ChunkSize = fs.SizeSuffix(3)
-
-	subFs2, err := fs.NewFs(ctx, subRemote+"/subdir2")
-	assert.NoError(t, err)
-	fs2, isChunkerFs := subFs2.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs2.useNoRename = true
-	fs2.opt.ChunkSize = fs.SizeSuffix(3)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	item := fstest.Item{Path: "movefile", ModTime: modTime}
-	contents := "abcdef"
-	_, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
-
-	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
-	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(len(contents)), dstFile.Size())
-
-	r, err := dstFile.Open(ctx)
-	assert.NoError(t, err)
-	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
-	assert.NoError(t, err)
-	assert.Equal(t, contents, string(data))
-	_ = r.Close()
-	_ = operations.Purge(ctx, f.base, dir)
-}
-
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("PutLarge", func(t *testing.T) {
@@ -870,12 +764,6 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("FutureProof", func(t *testing.T) {
 		testFutureProof(t, f)
 	})
-	t.Run("BackwardsCompatibility", func(t *testing.T) {
-		testBackwardsCompatibility(t, f)
-	})
-	t.Run("ChunkerServerSideMove", func(t *testing.T) {
-		testChunkerServerSideMove(t, f)
-	})
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/compress/compress.go

@@ -36,7 +36,7 @@ import (
 // Globals
 const (
 	initialChunkSize = 262144  // Initial and max sizes of chunks when reading parts of the file. Currently
-	maxChunkSize     = 8388608 // at 256 KiB and 8 MiB.
+	maxChunkSize     = 8388608 // at 256KB and 8 MB.
 
 	bufferSize          = 8388608
 	heuristicBytes      = 1048576
@@ -53,7 +53,7 @@ const (
 	Gzip         = 2
 )
 
-var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
+var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9+_]{11})$")
 
 // Register with Fs
 func init() {

backend/crypt/cipher.go

@@ -12,14 +12,12 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"unicode/utf8"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/crypto/scrypt"
@@ -444,32 +442,11 @@ func (c *Cipher) encryptFileName(in string) string {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets encrypted/obfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i] = c.encryptSegment(segments[i])
 		} else {
 			segments[i] = c.obfuscateSegment(segments[i])
 		}
-
-		// Add back a version to the encrypted/obfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/")
 }
@@ -500,21 +477,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets decrypted/deobfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i], err = c.decryptSegment(segments[i])
 		} else {
@@ -524,12 +486,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if err != nil {
 			return "", err
 		}
-
-		// Add back a version to the decrypted/deobfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/"), nil
 }
@@ -538,18 +494,10 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
 		remainingLength := len(in) - len(encryptedSuffix)
-		if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
-			return "", ErrorNotAnEncryptedFile
+		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
+			return in[:remainingLength], nil
 		}
-		decrypted := in[:remainingLength]
-		if version.Match(decrypted) {
-			_, unversioned := version.Remove(decrypted)
-			if unversioned == "" {
-				return "", ErrorNotAnEncryptedFile
-			}
-		}
-		// Leave the version string on, if it was there
-		return decrypted, nil
+		return "", ErrorNotAnEncryptedFile
 	}
 	return c.decryptFileName(in)
 }

backend/crypt/cipher_test.go

@@ -160,29 +160,22 @@ func TestEncryptFileName(t *testing.T) {
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Standard mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionStandard, "", "", false)
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Now off mode
 	c, _ = newCipher(NameEncryptionOff, "", "", true)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
-	assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
-	assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 	// Obfuscation mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
 	assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
-	assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 }
@@ -201,19 +194,14 @@ func TestDecryptFileName(t *testing.T) {
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
 		{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
 		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
 		{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
 		{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
 		{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
 		{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
 		{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
 	} {
 		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
 		actual, actualErr := c.DecryptFileName(test.in)

backend/crypt/crypt.go

@@ -101,21 +101,6 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
-		}, {
-			Name:     "no_data_encryption",
-			Help:     "Option to either encrypt file data or leave it unencrypted.",
-			Default:  false,
-			Advanced: true,
-			Examples: []fs.OptionExample{
-				{
-					Value: "true",
-					Help:  "Don't encrypt file data, leave it unencrypted.",
-				},
-				{
-					Value: "false",
-					Help:  "Encrypt file data.",
-				},
-			},
 		}},
 	})
 }
@@ -224,7 +209,6 @@ type Options struct {
 	Remote                  string `config:"remote"`
 	FilenameEncryption      string `config:"filename_encryption"`
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
-	NoDataEncryption        bool   `config:"no_data_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
@@ -362,10 +346,6 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
-	if f.opt.NoDataEncryption {
-		return put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
-	}
-
 	// Encrypt the data into wrappedIn
 	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
@@ -404,16 +384,13 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read destination hash")
 		}
-		if srcHash != "" && dstHash != "" {
-			if srcHash != dstHash {
-				// remove object
-				err = o.Remove(ctx)
-				if err != nil {
-					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
-				}
-				return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
+		if srcHash != "" && dstHash != "" && srcHash != dstHash {
+			// remove object
+			err = o.Remove(ctx)
+			if err != nil {
+				fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 			}
-			fs.Debugf(src, "%v = %s OK", ht, srcHash)
+			return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
 		}
 	}
 
@@ -640,10 +617,6 @@ func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Objec
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
-	if f.opt.NoDataEncryption {
-		return src.Hash(ctx, hashType)
-	}
-
 	// Read the nonce - opening the file is sufficient to read the nonce in
 	// use a limited read so we only read the header
 	in, err := o.Object.Open(ctx, &fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
@@ -849,13 +822,9 @@ func (o *Object) Remote() string {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	size := o.Object.Size()
-	if !o.f.opt.NoDataEncryption {
-		var err error
-		size, err = o.f.cipher.DecryptedSize(size)
-		if err != nil {
-			fs.Debugf(o, "Bad size for decrypt: %v", err)
-		}
+	size, err := o.f.cipher.DecryptedSize(o.Object.Size())
+	if err != nil {
+		fs.Debugf(o, "Bad size for decrypt: %v", err)
 	}
 	return size
 }
@@ -873,10 +842,6 @@ func (o *Object) UnWrap() fs.Object {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
-	if o.f.opt.NoDataEncryption {
-		return o.Object.Open(ctx, options...)
-	}
-
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
 	for _, option := range options {

backend/crypt/crypt_test.go

@@ -91,26 +91,3 @@ func TestObfuscate(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
-
-// TestNoDataObfuscate runs integration tests against the remote
-func TestNoDataObfuscate(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
-	name := "TestCrypt4"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*crypt.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "crypt"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
-			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
-			{Name: name, Key: "no_data_encryption", Value: "true"},
-		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-	})
-}

backend/drive/drive.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"mime"
 	"net/http"
 	"path"
@@ -67,8 +68,8 @@ const (
 	defaultScope                = "drive"
 	// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
-	minChunkSize     = 256 * fs.Kibi
-	defaultChunkSize = 8 * fs.Mebi
+	minChunkSize     = 256 * fs.KibiByte
+	defaultChunkSize = 8 * fs.MebiByte
 	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
@@ -182,71 +183,32 @@ func init() {
 		Description: "Google Drive",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
 			// Parse config into Options struct
 			opt := new(Options)
 			err := configstruct.Set(m, opt)
 			if err != nil {
-				return nil, errors.Wrap(err, "couldn't parse config into struct")
+				fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
+				return
 			}
 
-			switch config.State {
-			case "":
-				// Fill in the scopes
-				driveConfig.Scopes = driveScopes(opt.Scope)
-
-				// Set the root_folder_id if using drive.appfolder
-				if driveScopesContainsAppFolder(driveConfig.Scopes) {
-					m.Set("root_folder_id", "appDataFolder")
-				}
-
-				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
-					return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
-						OAuth2Config: driveConfig,
-					})
-				}
-				return fs.ConfigGoto("teamdrive")
-			case "teamdrive":
-				if opt.TeamDriveID == "" {
-					return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
-				}
-				return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
-			case "teamdrive_ok":
-				if config.Result == "false" {
-					m.Set("team_drive", "")
-					return nil, nil
-				}
-				return fs.ConfigGoto("teamdrive_config")
-			case "teamdrive_change":
-				if config.Result == "false" {
-					return nil, nil
-				}
-				return fs.ConfigGoto("teamdrive_config")
-			case "teamdrive_config":
-				f, err := newFs(ctx, name, "", m)
-				if err != nil {
-					return nil, errors.Wrap(err, "failed to make Fs to list Shared Drives")
-				}
-				teamDrives, err := f.listTeamDrives(ctx)
-				if err != nil {
-					return nil, err
-				}
-				if len(teamDrives) == 0 {
-					return fs.ConfigError("", "No Shared Drives found in your account")
-				}
-				return fs.ConfigChoose("teamdrive_final", "config_team_drive", "Shared Drive", len(teamDrives), func(i int) (string, string) {
-					teamDrive := teamDrives[i]
-					return teamDrive.Id, teamDrive.Name
-				})
-			case "teamdrive_final":
-				driveID := config.Result
-				m.Set("team_drive", driveID)
-				m.Set("root_folder_id", "")
-				opt.TeamDriveID = driveID
-				opt.RootFolderID = ""
-				return nil, nil
+			// Fill in the scopes
+			driveConfig.Scopes = driveScopes(opt.Scope)
+			// Set the root_folder_id if using drive.appfolder
+			if driveScopesContainsAppFolder(driveConfig.Scopes) {
+				m.Set("root_folder_id", "appDataFolder")
+			}
+
+			if opt.ServiceAccountFile == "" {
+				err = oauthutil.Config(ctx, "drive", name, m, driveConfig, nil)
+				if err != nil {
+					log.Fatalf("Failed to configure token: %v", err)
+				}
+			}
+			err = configTeamDrive(ctx, opt, m, name)
+			if err != nil {
+				log.Fatalf("Failed to configure Shared Drive: %v", err)
 			}
-			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
@@ -505,7 +467,7 @@ See: https://github.com/rclone/rclone/issues/3631
 			Default: false,
 			Help: `Make upload limit errors be fatal
 
-At the time of writing it is only possible to upload 750 GiB of data to
+At the time of writing it is only possible to upload 750GB of data to
 Google Drive a day (this is an undocumented limit). When this limit is
 reached Google Drive produces a slightly different error message. When
 this flag is set it causes these errors to be fatal.  These will stop
@@ -522,7 +484,7 @@ See: https://github.com/rclone/rclone/issues/3857
 			Default: false,
 			Help: `Make download limit errors be fatal
 
-At the time of writing it is only possible to download 10 TiB of data from
+At the time of writing it is only possible to download 10TB of data from
 Google Drive a day (this is an undocumented limit). When this limit is
 reached Google Drive produces a slightly different error message. When
 this flag is set it causes these errors to be fatal.  These will stop
@@ -560,7 +522,7 @@ If this flag is set then rclone will ignore shortcut files completely.
 	} {
 		for mimeType, extension := range m {
 			if err := mime.AddExtensionType(extension, mimeType); err != nil {
-				fs.Errorf("Failed to register MIME type %q: %v", mimeType, err)
+				log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
 			}
 		}
 	}
@@ -628,13 +590,13 @@ type Fs struct {
 }
 
 type baseObject struct {
-	fs           *Fs      // what this object is part of
-	remote       string   // The remote path
-	id           string   // Drive Id of this object
-	modifiedDate string   // RFC3339 time it was last modified
-	mimeType     string   // The object MIME type
-	bytes        int64    // size of the object
-	parents      []string // IDs of the parent directories
+	fs           *Fs    // what this object is part of
+	remote       string // The remote path
+	id           string // Drive Id of this object
+	modifiedDate string // RFC3339 time it was last modified
+	mimeType     string // The object MIME type
+	bytes        int64  // size of the object
+	parents      int    // number of parents
 }
 type documentObject struct {
 	baseObject
@@ -679,10 +641,7 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
 	}
@@ -736,20 +695,20 @@ func containsString(slice []string, s string) bool {
 }
 
 // getFile returns drive.File for the ID passed and fields passed in
-func (f *Fs) getFile(ctx context.Context, ID string, fields googleapi.Field) (info *drive.File, err error) {
+func (f *Fs) getFile(ID string, fields googleapi.Field) (info *drive.File, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Get(ID).
 			Fields(fields).
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	return info, err
 }
 
 // getRootID returns the canonical ID for the "root" ID
-func (f *Fs) getRootID(ctx context.Context) (string, error) {
-	info, err := f.getFile(ctx, "root", "id")
+func (f *Fs) getRootID() (string, error) {
+	info, err := f.getFile("root", "id")
 	if err != nil {
 		return "", errors.Wrap(err, "couldn't find root directory ID")
 	}
@@ -855,7 +814,7 @@ OUTER:
 		var files *drive.FileList
 		err = f.pacer.Call(func() (bool, error) {
 			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return false, errors.Wrap(err, "couldn't list directory")
@@ -878,7 +837,7 @@ OUTER:
 				if filesOnly && item.ShortcutDetails.TargetMimeType == driveFolderType {
 					continue
 				}
-				item, err = f.resolveShortcut(ctx, item)
+				item, err = f.resolveShortcut(item)
 				if err != nil {
 					return false, errors.Wrap(err, "list")
 				}
@@ -896,7 +855,7 @@ OUTER:
 				if !found {
 					continue
 				}
-				_, exportName, _, _ := f.findExportFormat(ctx, item)
+				_, exportName, _, _ := f.findExportFormat(item)
 				if exportName == "" || exportName != title {
 					continue
 				}
@@ -987,6 +946,48 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
 	return
 }
 
+// Figure out if the user wants to use a team drive
+func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name string) error {
+	ci := fs.GetConfig(ctx)
+
+	// Stop if we are running non-interactive config
+	if ci.AutoConfirm {
+		return nil
+	}
+	if opt.TeamDriveID == "" {
+		fmt.Printf("Configure this as a Shared Drive (Team Drive)?\n")
+	} else {
+		fmt.Printf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID)
+	}
+	if !config.Confirm(false) {
+		return nil
+	}
+	f, err := newFs(ctx, name, "", m)
+	if err != nil {
+		return errors.Wrap(err, "failed to make Fs to list Shared Drives")
+	}
+	fmt.Printf("Fetching Shared Drive list...\n")
+	teamDrives, err := f.listTeamDrives(ctx)
+	if err != nil {
+		return err
+	}
+	if len(teamDrives) == 0 {
+		fmt.Printf("No Shared Drives found in your account")
+		return nil
+	}
+	var driveIDs, driveNames []string
+	for _, teamDrive := range teamDrives {
+		driveIDs = append(driveIDs, teamDrive.Id)
+		driveNames = append(driveNames, teamDrive.Name)
+	}
+	driveID := config.Choose("Enter a Shared Drive ID", driveIDs, driveNames, true)
+	m.Set("team_drive", driveID)
+	m.Set("root_folder_id", "")
+	opt.TeamDriveID = driveID
+	opt.RootFolderID = ""
+	return nil
+}
+
 // getClient makes an http client according to the options
 func getClient(ctx context.Context, opt *Options) *http.Client {
 	t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
@@ -1154,7 +1155,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 		f.rootFolderID = f.opt.TeamDriveID
 	} else {
 		// otherwise look up the actual root ID
-		rootID, err := f.getRootID(ctx)
+		rootID, err := f.getRootID()
 		if err != nil {
 			if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
 				// 404 means that this scope does not have permission to get the
@@ -1165,7 +1166,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 			}
 		}
 		f.rootFolderID = rootID
-		fs.Debugf(f, "'root_folder_id = %s' - save this in the config to speed up startup", rootID)
+		fs.Debugf(f, "root_folder_id = %q - save this in the config to speed up startup", rootID)
 	}
 
 	f.dirCache = dircache.New(f.root, f.rootFolderID, f)
@@ -1235,7 +1236,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 		modifiedDate: modifiedDate,
 		mimeType:     info.MimeType,
 		bytes:        size,
-		parents:      info.Parents,
+		parents:      len(info.Parents),
 	}
 }
 
@@ -1327,26 +1328,26 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
 // newObjectWithInfo creates an fs.Object for any drive.File
 //
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
-	// If item has MD5 sum it is a file stored on drive
-	if info.Md5Checksum != "" {
+func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
+	// If item has MD5 sum or a length it is a file stored on drive
+	if info.Md5Checksum != "" || info.Size > 0 {
 		return f.newRegularObject(remote, info), nil
 	}
 
-	extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
-	return f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
+	extension, exportName, exportMimeType, isDocument := f.findExportFormat(info)
+	return f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
 }
 
 // newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
 //
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
-	ctx context.Context, remote string, info *drive.File,
+	remote string, info *drive.File,
 	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
-	info, err = f.resolveShortcut(ctx, info)
+	info, err = f.resolveShortcut(info)
 	if err != nil {
 		return nil, errors.Wrap(err, "new object")
 	}
@@ -1362,8 +1363,8 @@ func (f *Fs) newObjectWithExportInfo(
 		// Pretend a dangling shortcut is a regular object
 		// It will error if used, but appear in listings so it can be deleted
 		return f.newRegularObject(remote, info), nil
-	case info.Md5Checksum != "":
-		// If item has MD5 sum it is a file stored on drive
+	case info.Md5Checksum != "" || info.Size > 0:
+		// If item has MD5 sum or a length it is a file stored on drive
 		return f.newRegularObject(remote, info), nil
 	case f.opt.SkipGdocs:
 		fs.Debugf(remote, "Skipping google document type %q", info.MimeType)
@@ -1394,7 +1395,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	}
 
 	remote = remote[:len(remote)-len(extension)]
-	obj, err := f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
+	obj, err := f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
 	switch {
 	case err != nil:
 		return nil, err
@@ -1411,7 +1412,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	pathID = actualID(pathID)
 	found, err = f.list(ctx, []string{pathID}, leaf, true, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
 		if !f.opt.SkipGdocs {
-			_, exportName, _, isDocument := f.findExportFormat(ctx, item)
+			_, exportName, _, isDocument := f.findExportFormat(item)
 			if exportName == leaf {
 				pathIDOut = item.Id
 				return true
@@ -1446,8 +1447,8 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		info, err = f.svc.Files.Create(createInfo).
 			Fields("id").
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return "", err
@@ -1482,15 +1483,15 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
-func (f *Fs) fetchFormats(ctx context.Context) {
+func (f *Fs) fetchFormats() {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			about, err = f.svc.About.Get().
 				Fields("exportFormats,importFormats").
-				Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
+				Do()
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			fs.Errorf(f, "Failed to get Drive exportFormats and importFormats: %v", err)
@@ -1507,8 +1508,8 @@ func (f *Fs) fetchFormats(ctx context.Context) {
 // if necessary.
 //
 // if the fetch fails then it will not export any drive formats
-func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
-	f.fetchFormats(ctx)
+func (f *Fs) exportFormats() map[string][]string {
+	f.fetchFormats()
 	return _exportFormats
 }
 
@@ -1516,8 +1517,8 @@ func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
 // if necessary.
 //
 // if the fetch fails then it will not import any drive formats
-func (f *Fs) importFormats(ctx context.Context) map[string][]string {
-	f.fetchFormats(ctx)
+func (f *Fs) importFormats() map[string][]string {
+	f.fetchFormats()
 	return _importFormats
 }
 
@@ -1526,9 +1527,9 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 //
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
-func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
+func (f *Fs) findExportFormatByMimeType(itemMimeType string) (
 	extension, mimeType string, isDocument bool) {
-	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
+	exportMimeTypes, isDocument := f.exportFormats()[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
 			_mimeType := mime.TypeByExtension(_extension)
@@ -1555,8 +1556,8 @@ func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string
 //
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", "", false)
-func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension, filename, mimeType string, isDocument bool) {
-	extension, mimeType, isDocument = f.findExportFormatByMimeType(ctx, item.MimeType)
+func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
+	extension, mimeType, isDocument = f.findExportFormatByMimeType(item.MimeType)
 	if extension != "" {
 		filename = item.Name + extension
 	}
@@ -1568,9 +1569,9 @@ func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension,
 // MIME type is returned
 //
 // When no match is found "" is returned.
-func (f *Fs) findImportFormat(ctx context.Context, mimeType string) string {
+func (f *Fs) findImportFormat(mimeType string) string {
 	mimeType = fixMimeType(mimeType)
-	ifs := f.importFormats(ctx)
+	ifs := f.importFormats()
 	for _, mt := range f.importMimeTypes {
 		if mt == mimeType {
 			importMimeTypes := ifs[mimeType]
@@ -1603,7 +1604,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	var iErr error
 	_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
-		entry, err := f.itemToDirEntry(ctx, path.Join(dir, item.Name), item)
+		entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item)
 		if err != nil {
 			iErr = err
 			return true
@@ -1716,7 +1717,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 					}
 				}
 				remote := path.Join(paths[i], item.Name)
-				entry, err := f.itemToDirEntry(ctx, remote, item)
+				entry, err := f.itemToDirEntry(remote, item)
 				if err != nil {
 					iErr = err
 					return true
@@ -1981,7 +1982,7 @@ func isShortcut(item *drive.File) bool {
 // Note that we assume shortcuts can't point to shortcuts. Google
 // drive web interface doesn't offer the option to create a shortcut
 // to a shortcut. The documentation is silent on the issue.
-func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *drive.File, err error) {
+func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error) {
 	if f.opt.SkipShortcuts || item.MimeType != shortcutMimeType {
 		return item, nil
 	}
@@ -1989,7 +1990,7 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
 		fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
 		return item, nil
 	}
-	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
+	newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
 		if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
 			// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
@@ -2011,21 +2012,18 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
 // itemToDirEntry converts a drive.File to an fs.DirEntry.
 // When the drive.File cannot be represented as an fs.DirEntry
 // (nil, nil) is returned.
-func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
+func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry, err error) {
 	switch {
 	case item.MimeType == driveFolderType:
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, item.Id)
 		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
 		d := fs.NewDir(remote, when).SetID(item.Id)
-		if len(item.Parents) > 0 {
-			d.SetParentID(item.Parents[0])
-		}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
 	default:
-		entry, err = f.newObjectWithInfo(ctx, remote, item)
+		entry, err = f.newObjectWithInfo(remote, item)
 		if err == fs.ErrorObjectNotFound {
 			return nil, nil
 		}
@@ -2092,12 +2090,12 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	importMimeType := ""
 
 	if f.importMimeTypes != nil && !f.opt.SkipGdocs {
-		importMimeType = f.findImportFormat(ctx, srcMimeType)
+		importMimeType = f.findImportFormat(srcMimeType)
 
 		if isInternalMimeType(importMimeType) {
 			remote = remote[:len(remote)-len(srcExt)]
 
-			exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
+			exportExt, _, _ = f.findExportFormatByMimeType(importMimeType)
 			if exportExt == "" {
 				return nil, errors.Errorf("No export format found for %q", importMimeType)
 			}
@@ -2127,8 +2125,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 				Fields(partialFields).
 				SupportsAllDrives(true).
 				KeepRevisionForever(f.opt.KeepRevisionForever).
-				Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
+				Do()
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return nil, err
@@ -2140,7 +2138,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 			return nil, err
 		}
 	}
-	return f.newObjectWithInfo(ctx, remote, info)
+	return f.newObjectWithInfo(remote, info)
 }
 
 // MergeDirs merges the contents of all the directories passed
@@ -2182,8 +2180,8 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 					AddParents(dstDir.ID()).
 					Fields("").
 					SupportsAllDrives(true).
-					Context(ctx).Do()
-				return f.shouldRetry(ctx, err)
+					Do()
+				return f.shouldRetry(err)
 			})
 			if err != nil {
 				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
@@ -2216,14 +2214,14 @@ func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 			_, err = f.svc.Files.Update(id, &info).
 				Fields("").
 				SupportsAllDrives(true).
-				Context(ctx).Do()
+				Do()
 		} else {
 			err = f.svc.Files.Delete(id).
 				Fields("").
 				SupportsAllDrives(true).
-				Context(ctx).Do()
+				Do()
 		}
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 }
 
@@ -2336,12 +2334,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	if isDoc {
 		// preserve the description on copy for docs
-		info, err := f.getFile(ctx, actualID(srcObj.id), "description")
+		info, err := f.getFile(actualID(srcObj.id), "description")
 		if err != nil {
-			fs.Errorf(srcObj, "Failed to read description for Google Doc: %v", err)
-		} else {
-			createInfo.Description = info.Description
+			return nil, errors.Wrap(err, "failed to read description for Google Doc")
 		}
+		createInfo.Description = info.Description
 	} else {
 		// don't overwrite the description on copy for files
 		// this should work for docs but it doesn't - it is probably a bug in Google Drive
@@ -2357,13 +2354,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
 	}
-	newObject, err := f.newObjectWithInfo(ctx, remote, info)
+	newObject, err := f.newObjectWithInfo(remote, info)
 	if err != nil {
 		return nil, err
 	}
@@ -2457,7 +2454,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 
 	if err != nil {
@@ -2475,7 +2472,7 @@ func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
 	var td *drive.Drive
 	err = f.pacer.Call(func() (bool, error) {
 		td, err = f.svc.Drives.Get(f.opt.TeamDriveID).Fields("name,id,capabilities,createdTime,restrictions").Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get Shared Drive info")
@@ -2498,7 +2495,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		about, err = f.svc.About.Get().Fields("storageQuota").Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get Drive storageQuota")
@@ -2570,14 +2567,14 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			AddParents(dstParents).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	return f.newObjectWithInfo(ctx, remote, info)
+	return f.newObjectWithInfo(remote, info)
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
@@ -2607,8 +2604,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		_, err = f.svc.Permissions.Create(id, permission).
 			Fields("").
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return "", err
@@ -2650,8 +2647,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			AddParents(dstDirectoryID).
 			Fields("").
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+			Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -2669,7 +2666,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
 	go func() {
 		// get the StartPageToken early so all changes from now on get processed
-		startPageToken, err := f.changeNotifyStartPageToken(ctx)
+		startPageToken, err := f.changeNotifyStartPageToken()
 		if err != nil {
 			fs.Infof(f, "Failed to get StartPageToken: %s", err)
 		}
@@ -2694,7 +2691,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 				}
 			case <-tickerC:
 				if startPageToken == "" {
-					startPageToken, err = f.changeNotifyStartPageToken(ctx)
+					startPageToken, err = f.changeNotifyStartPageToken()
 					if err != nil {
 						fs.Infof(f, "Failed to get StartPageToken: %s", err)
 						continue
@@ -2709,15 +2706,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
-func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
+func (f *Fs) changeNotifyStartPageToken() (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
 		changes := f.svc.Changes.GetStartPageToken().SupportsAllDrives(true)
 		if f.isTeamDrive {
 			changes.DriveId(f.opt.TeamDriveID)
 		}
-		startPageToken, err = changes.Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
+		startPageToken, err = changes.Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return
@@ -2746,7 +2743,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 				changesCall.Spaces("appDataFolder")
 			}
 			changeList, err = changesCall.Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return
@@ -2942,8 +2939,8 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(dstFs.opt.KeepRevisionForever).
-			Context(ctx).Do()
-		return dstFs.shouldRetry(ctx, err)
+			Do()
+		return dstFs.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "shortcut creation failed")
@@ -2951,24 +2948,24 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 	if isDir {
 		return nil, nil
 	}
-	return dstFs.newObjectWithInfo(ctx, dstPath, info)
+	return dstFs.newObjectWithInfo(dstPath, info)
 }
 
 // List all team drives
-func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.Drive, err error) {
-	drives = []*drive.Drive{}
-	listTeamDrives := f.svc.Drives.List().PageSize(100)
+func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err error) {
+	drives = []*drive.TeamDrive{}
+	listTeamDrives := f.svc.Teamdrives.List().PageSize(100)
 	var defaultFs Fs // default Fs with default Options
 	for {
-		var teamDrives *drive.DriveList
+		var teamDrives *drive.TeamDriveList
 		err = f.pacer.Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Context(ctx).Do()
-			return defaultFs.shouldRetry(ctx, err)
+			return defaultFs.shouldRetry(err)
 		})
 		if err != nil {
 			return drives, errors.Wrap(err, "listing Team Drives failed")
 		}
-		drives = append(drives, teamDrives.Drives...)
+		drives = append(drives, teamDrives.TeamDrives...)
 		if teamDrives.NextPageToken == "" {
 			break
 		}
@@ -3005,8 +3002,8 @@ func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurs
 				_, err := f.svc.Files.Update(item.Id, &update).
 					SupportsAllDrives(true).
 					Fields("trashed").
-					Context(ctx).Do()
-				return f.shouldRetry(ctx, err)
+					Do()
+				return f.shouldRetry(err)
 			})
 			if err != nil {
 				err = errors.Wrap(err, "failed to restore")
@@ -3048,7 +3045,7 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 
 // copy file with id to dest
 func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
-	info, err := f.getFile(ctx, id, f.fileFields)
+	info, err := f.getFile(id, f.fileFields)
 	if err != nil {
 		return errors.Wrap(err, "couldn't find id")
 	}
@@ -3056,7 +3053,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 		return errors.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
 	}
 	info.Name = f.opt.Enc.ToStandardName(info.Name)
-	o, err := f.newObjectWithInfo(ctx, info.Name, info)
+	o, err := f.newObjectWithInfo(info.Name, info)
 	if err != nil {
 		return err
 	}
@@ -3065,7 +3062,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 		return err
 	}
 	if destLeaf == "" {
-		destLeaf = path.Base(o.Remote())
+		destLeaf = info.Name
 	}
 	if destDir == "" {
 		destDir = "."
@@ -3357,7 +3354,7 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 
 	found, err := f.list(ctx, []string{directoryID}, leaf, false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
 		if !f.opt.SkipGdocs {
-			extension, exportName, exportMimeType, isDocument = f.findExportFormat(ctx, item)
+			extension, exportName, exportMimeType, isDocument = f.findExportFormat(item)
 			if exportName == leaf {
 				info = item
 				return true
@@ -3408,8 +3405,8 @@ func (o *baseObject) SetModTime(ctx context.Context, modTime time.Time) error {
 		info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return o.fs.shouldRetry(ctx, err)
+			Do()
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -3430,10 +3427,11 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 	if url == "" {
 		return nil, nil, errors.New("forbidden to download - check sharing permission")
 	}
-	req, err = http.NewRequestWithContext(ctx, method, url, nil)
+	req, err = http.NewRequest(method, url, nil)
 	if err != nil {
 		return req, nil, err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	if o.bytes == 0 {
 		// Don't supply range requests for 0 length objects as they always fail
@@ -3447,7 +3445,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return o.fs.shouldRetry(ctx, err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return req, nil, err
@@ -3539,8 +3537,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			v2File, err = o.fs.v2Svc.Files.Get(actualID(o.id)).
 				Fields("downloadUrl").
 				SupportsAllDrives(true).
-				Context(ctx).Do()
-			return o.fs.shouldRetry(ctx, err)
+				Do()
+			return o.fs.shouldRetry(err)
 		})
 		if err == nil {
 			fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
@@ -3620,8 +3618,8 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 				Fields(partialFields).
 				SupportsAllDrives(true).
 				KeepRevisionForever(o.fs.opt.KeepRevisionForever).
-				Context(ctx).Do()
-			return o.fs.shouldRetry(ctx, err)
+				Do()
+			return o.fs.shouldRetry(err)
 		})
 		return
 	}
@@ -3664,7 +3662,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
+	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
 	if err != nil {
 		return err
 	}
@@ -3688,7 +3686,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
 	if o.fs.importMimeTypes == nil || o.fs.opt.SkipGdocs {
 		return errors.Errorf("can't update google document type without --drive-import-formats")
 	}
-	importMimeType = o.fs.findImportFormat(ctx, updateInfo.MimeType)
+	importMimeType = o.fs.findImportFormat(updateInfo.MimeType)
 	if importMimeType == "" {
 		return errors.Errorf("no import format found for %q", srcMimeType)
 	}
@@ -3705,7 +3703,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
 	remote := src.Remote()
 	remote = remote[:len(remote)-o.extLen]
 
-	newO, err := o.fs.newObjectWithInfo(ctx, remote, info)
+	newO, err := o.fs.newObjectWithInfo(remote, info)
 	if err != nil {
 		return err
 	}
@@ -3725,7 +3723,7 @@ func (o *linkObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo
 
 // Remove an object
 func (o *baseObject) Remove(ctx context.Context) error {
-	if len(o.parents) > 1 {
+	if o.parents > 1 {
 		return errors.New("can't delete safely - has multiple parents")
 	}
 	return o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
@@ -3741,14 +3739,6 @@ func (o *baseObject) ID() string {
 	return o.id
 }
 
-// ParentID returns the ID of the Object parent if known, or "" if not
-func (o *baseObject) ParentID() string {
-	if len(o.parents) > 0 {
-		return o.parents[0]
-	}
-	return ""
-}
-
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
@@ -3809,13 +3799,10 @@ var (
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
-	_ fs.ParentIDer      = (*Object)(nil)
 	_ fs.Object          = (*documentObject)(nil)
 	_ fs.MimeTyper       = (*documentObject)(nil)
 	_ fs.IDer            = (*documentObject)(nil)
-	_ fs.ParentIDer      = (*documentObject)(nil)
 	_ fs.Object          = (*linkObject)(nil)
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
-	_ fs.ParentIDer      = (*linkObject)(nil)
 )

backend/drive/drive_internal_test.go

@@ -111,7 +111,6 @@ func TestInternalParseExtensions(t *testing.T) {
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
-	ctx := context.Background()
 	item := &drive.File{
 		Name:     "file",
 		MimeType: "application/vnd.google-apps.document",
@@ -129,7 +128,7 @@ func TestInternalFindExportFormat(t *testing.T) {
 	} {
 		f := new(Fs)
 		f.exportExtensions = test.extensions
-		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
+		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
 		assert.Equal(t, test.wantExtension, gotExtension)
 		if test.wantExtension != "" {
 			assert.Equal(t, item.Name+gotExtension, gotFilename)

backend/drive/upload.go

@@ -77,10 +77,11 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 			return false, err
 		}
 		var req *http.Request
-		req, err = http.NewRequestWithContext(ctx, method, urls, body)
+		req, err = http.NewRequest(method, urls, body)
 		if err != nil {
 			return false, err
 		}
+		req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 		googleapi.Expand(req.URL, map[string]string{
 			"fileId": fileID,
 		})
@@ -94,7 +95,7 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -113,7 +114,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 
 // Make an http.Request for the range passed in
 func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
-	req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
+	req, _ := http.NewRequest("POST", rx.URI, body)
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	req.ContentLength = reqSize
 	totalSize := "*"
 	if rx.ContentLength >= 0 {
@@ -202,7 +204,7 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
 			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
-			again, err := rx.f.shouldRetry(ctx, err)
+			again, err := rx.f.shouldRetry(err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/batcher.go

@@ -1,350 +0,0 @@
-// This file contains the implementation of the sync batcher for uploads
-//
-// Dropbox rules say you can start as many batches as you want, but
-// you may only have one batch being committed and must wait for the
-// batch to be finished before committing another.
-
-package dropbox
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/async"
-	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/atexit"
-)
-
-const (
-	maxBatchSize          = 1000                   // max size the batch can be
-	defaultTimeoutSync    = 500 * time.Millisecond // kick off the batch if nothing added for this long (sync)
-	defaultTimeoutAsync   = 10 * time.Second       // kick off the batch if nothing added for this long (ssync)
-	defaultBatchSizeAsync = 100                    // default batch size if async
-)
-
-// batcher holds info about the current items waiting for upload
-type batcher struct {
-	f        *Fs                 // Fs this batch is part of
-	mode     string              // configured batch mode
-	size     int                 // maximum size for batch
-	timeout  time.Duration       // idle timeout for batch
-	async    bool                // whether we are using async batching
-	in       chan batcherRequest // incoming items to batch
-	closed   chan struct{}       // close to indicate batcher shut down
-	atexit   atexit.FnHandle     // atexit handle
-	shutOnce sync.Once           // make sure we shutdown once only
-	wg       sync.WaitGroup      // wait for shutdown
-}
-
-// batcherRequest holds an incoming request with a place for a reply
-type batcherRequest struct {
-	commitInfo *files.UploadSessionFinishArg
-	result     chan<- batcherResponse
-}
-
-// Return true if batcherRequest is the quit request
-func (br *batcherRequest) isQuit() bool {
-	return br.commitInfo == nil
-}
-
-// Send this to get the engine to quit
-var quitRequest = batcherRequest{}
-
-// batcherResponse holds a response to be delivered to clients waiting
-// for a batch to complete.
-type batcherResponse struct {
-	err   error
-	entry *files.FileMetadata
-}
-
-// newBatcher creates a new batcher structure
-func newBatcher(ctx context.Context, f *Fs, mode string, size int, timeout time.Duration) (*batcher, error) {
-	// fs.Debugf(f, "Creating batcher with mode %q, size %d, timeout %v", mode, size, timeout)
-	if size > maxBatchSize || size < 0 {
-		return nil, errors.Errorf("dropbox: batch size must be < %d and >= 0 - it is currently %d", maxBatchSize, size)
-	}
-
-	async := false
-
-	switch mode {
-	case "sync":
-		if size <= 0 {
-			ci := fs.GetConfig(ctx)
-			size = ci.Transfers
-		}
-		if timeout <= 0 {
-			timeout = defaultTimeoutSync
-		}
-	case "async":
-		if size <= 0 {
-			size = defaultBatchSizeAsync
-		}
-		if timeout <= 0 {
-			timeout = defaultTimeoutAsync
-		}
-		async = true
-	case "off":
-		size = 0
-	default:
-		return nil, errors.Errorf("dropbox: batch mode must be sync|async|off not %q", mode)
-	}
-
-	b := &batcher{
-		f:       f,
-		mode:    mode,
-		size:    size,
-		timeout: timeout,
-		async:   async,
-		in:      make(chan batcherRequest, size),
-		closed:  make(chan struct{}),
-	}
-	if b.Batching() {
-		b.atexit = atexit.Register(b.Shutdown)
-		b.wg.Add(1)
-		go b.commitLoop(context.Background())
-	}
-	return b, nil
-
-}
-
-// Batching returns true if batching is active
-func (b *batcher) Batching() bool {
-	return b.size > 0
-}
-
-// finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
-	var arg = &files.UploadSessionFinishBatchArg{
-		Entries: items,
-	}
-	err = b.f.pacer.Call(func() (bool, error) {
-		batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
-		// If error is insufficient space then don't retry
-		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
-			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
-				err = fserrors.NoRetryError(err)
-				return false, err
-			}
-		}
-		// after the first chunk is uploaded, we retry everything
-		return err != nil, err
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "batch commit failed")
-	}
-	return batchStatus, nil
-}
-
-// finishBatchJobStatus waits for the batch to complete returning completed entries
-func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *files.UploadSessionFinishBatchLaunch) (complete *files.UploadSessionFinishBatchResult, err error) {
-	if launchBatchStatus.AsyncJobId == "" {
-		return nil, errors.New("wait for batch completion: empty job ID")
-	}
-	var batchStatus *files.UploadSessionFinishBatchJobStatus
-	sleepTime := 100 * time.Millisecond
-	const maxTries = 120
-	for try := 1; try <= maxTries; try++ {
-		err = b.f.pacer.Call(func() (bool, error) {
-			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
-				AsyncJobId: launchBatchStatus.AsyncJobId,
-			})
-			return shouldRetry(ctx, err)
-		})
-		if err != nil {
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d/%d", sleepTime, err, try, maxTries)
-		} else {
-			if batchStatus.Tag == "complete" {
-				return batchStatus.Complete, nil
-			}
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d/%d", sleepTime, batchStatus.Tag, try, maxTries)
-		}
-		time.Sleep(sleepTime)
-		sleepTime *= 2
-		if sleepTime > time.Second {
-			sleepTime = time.Second
-		}
-	}
-	if err == nil {
-		err = errors.New("batch didn't complete")
-	}
-	return nil, errors.Wrapf(err, "wait for batch failed after %d tries", maxTries)
-}
-
-// commit a batch
-func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []chan<- batcherResponse) (err error) {
-	// If commit fails then signal clients if sync
-	var signalled = b.async
-	defer func() {
-		if err != nil && signalled {
-			// Signal to clients that there was an error
-			for _, result := range results {
-				result <- batcherResponse{err: err}
-			}
-		}
-	}()
-	desc := fmt.Sprintf("%s batch length %d starting with: %s", b.mode, len(items), items[0].Commit.Path)
-	fs.Debugf(b.f, "Committing %s", desc)
-
-	// finalise the batch getting either a result or a job id to poll
-	batchStatus, err := b.finishBatch(ctx, items)
-	if err != nil {
-		return err
-	}
-
-	// check whether batch is complete
-	var complete *files.UploadSessionFinishBatchResult
-	switch batchStatus.Tag {
-	case "async_job_id":
-		// wait for batch to complete
-		complete, err = b.finishBatchJobStatus(ctx, batchStatus)
-		if err != nil {
-			return err
-		}
-	case "complete":
-		complete = batchStatus.Complete
-	default:
-		return errors.Errorf("batch returned unknown status %q", batchStatus.Tag)
-	}
-
-	// Check we got the right number of entries
-	entries := complete.Entries
-	if len(entries) != len(results) {
-		return errors.Errorf("expecting %d items in batch but got %d", len(results), len(entries))
-	}
-
-	// Report results to clients
-	var (
-		errorTag   = ""
-		errorCount = 0
-	)
-	for i := range results {
-		item := entries[i]
-		resp := batcherResponse{}
-		if item.Tag == "success" {
-			resp.entry = item.Success
-		} else {
-			errorCount++
-			errorTag = item.Tag
-			if item.Failure != nil {
-				errorTag = item.Failure.Tag
-				if item.Failure.LookupFailed != nil {
-					errorTag += "/" + item.Failure.LookupFailed.Tag
-				}
-				if item.Failure.Path != nil {
-					errorTag += "/" + item.Failure.Path.Tag
-				}
-				if item.Failure.PropertiesError != nil {
-					errorTag += "/" + item.Failure.PropertiesError.Tag
-				}
-			}
-			resp.err = errors.Errorf("batch upload failed: %s", errorTag)
-		}
-		if !b.async {
-			results[i] <- resp
-		}
-	}
-	// Show signalled so no need to report error to clients from now on
-	signalled = true
-
-	// Report an error if any failed in the batch
-	if errorTag != "" {
-		return errors.Errorf("batch had %d errors: last error: %s", errorCount, errorTag)
-	}
-
-	fs.Debugf(b.f, "Committed %s", desc)
-	return nil
-}
-
-// commitLoop runs the commit engine in the background
-func (b *batcher) commitLoop(ctx context.Context) {
-	var (
-		items     []*files.UploadSessionFinishArg // current batch of uncommitted files
-		results   []chan<- batcherResponse        // current batch of clients awaiting results
-		idleTimer = time.NewTimer(b.timeout)
-		commit    = func() {
-			err := b.commitBatch(ctx, items, results)
-			if err != nil {
-				fs.Errorf(b.f, "%s batch commit: failed to commit batch length %d: %v", b.mode, len(items), err)
-			}
-			items, results = nil, nil
-		}
-	)
-	defer b.wg.Done()
-	defer idleTimer.Stop()
-	idleTimer.Stop()
-
-outer:
-	for {
-		select {
-		case req := <-b.in:
-			if req.isQuit() {
-				break outer
-			}
-			items = append(items, req.commitInfo)
-			results = append(results, req.result)
-			idleTimer.Stop()
-			if len(items) >= b.size {
-				commit()
-			} else {
-				idleTimer.Reset(b.timeout)
-			}
-		case <-idleTimer.C:
-			if len(items) > 0 {
-				fs.Debugf(b.f, "Batch idle for %v so committing", b.timeout)
-				commit()
-			}
-		}
-
-	}
-	// commit any remaining items
-	if len(items) > 0 {
-		commit()
-	}
-}
-
-// Shutdown finishes any pending batches then shuts everything down
-//
-// Can be called from atexit handler
-func (b *batcher) Shutdown() {
-	b.shutOnce.Do(func() {
-		atexit.Unregister(b.atexit)
-		fs.Infof(b.f, "Commiting uploads - please wait...")
-		// show that batcher is shutting down
-		close(b.closed)
-		// quit the commitLoop by sending a quitRequest message
-		//
-		// Note that we don't close b.in because that will
-		// cause write to closed channel in Commit when we are
-		// exiting due to a signal.
-		b.in <- quitRequest
-		b.wg.Wait()
-	})
-}
-
-// Commit commits the file using a batch call, first adding it to the
-// batch and then waiting for the batch to complete in a synchronous
-// way if async is not set.
-func (b *batcher) Commit(ctx context.Context, commitInfo *files.UploadSessionFinishArg) (entry *files.FileMetadata, err error) {
-	select {
-	case <-b.closed:
-		return nil, fserrors.FatalError(errors.New("batcher is shutting down"))
-	default:
-	}
-	fs.Debugf(b.f, "Adding %q to batch", commitInfo.Commit.Path)
-	resp := make(chan batcherResponse, 1)
-	b.in <- batcherRequest{
-		commitInfo: commitInfo,
-		result:     resp,
-	}
-	// If running async then don't wait for the result
-	if b.async {
-		return nil, nil
-	}
-	result := <-resp
-	return result.entry, result.err
-}

backend/fichier/api.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"io"
 	"net/http"
-	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
@@ -29,10 +28,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	// Detect this error which the integration tests provoke
 	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
 	//
@@ -52,51 +48,10 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 
 var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
 
-func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) {
-	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
-	if err != nil {
-		return
-	}
-	// Temporary Object under construction
-	o = &Object{
-		fs:     f,
-		remote: remote,
-	}
-	return o, leaf, directoryID, nil
-}
-
-func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
-	request := FileInfoRequest{
-		URL: url,
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/info.cgi",
-	}
-
-	var file File
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't read file info")
-	}
-
-	return &file, err
-}
-
-// maybe do some actual validation later if necessary
-func validToken(token *GetTokenResponse) bool {
-	return token.Status == "OK"
-}
-
 func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
 	request := DownloadRequest{
 		URL:    url,
 		Single: 1,
-		Pass:   f.opt.FilePassword,
 	}
 	opts := rest.Opts{
 		Method: "POST",
@@ -106,8 +61,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
 	var token GetTokenResponse
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
-		doretry, err := shouldRetry(ctx, resp, err)
-		return doretry || !validToken(&token), err
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -126,22 +80,16 @@ func fileFromSharedFile(file *SharedFile) File {
 
 func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) {
 	opts := rest.Opts{
-		Method:      "GET",
-		RootURL:     "https://1fichier.com/dir/",
-		Path:        id,
-		Parameters:  map[string][]string{"json": {"1"}},
-		ContentType: "application/x-www-form-urlencoded",
-	}
-	if f.opt.FolderPassword != "" {
-		opts.Method = "POST"
-		opts.Parameters = nil
-		opts.Body = strings.NewReader("json=1&pass=" + url.QueryEscape(f.opt.FolderPassword))
+		Method:     "GET",
+		RootURL:    "https://1fichier.com/dir/",
+		Path:       id,
+		Parameters: map[string][]string{"json": {"1"}},
 	}
 
 	var sharedFiles SharedFolderResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -170,7 +118,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
 	filesList = &FilesList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
@@ -198,7 +146,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 	foldersList = &FoldersList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list folders")
@@ -292,7 +240,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons
 	response = &MakeFolderResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create folder")
@@ -319,13 +267,13 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't remove folder")
 	}
 	if response.Status != "OK" {
-		return nil, errors.Errorf("can't remove folder: %s", response.Message)
+		return nil, errors.New("Can't remove non-empty dir")
 	}
 
 	// fs.Debugf(f, "Removed Folder with id `%s`", directoryID)
@@ -348,7 +296,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
 	response = &GenericOKResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if err != nil {
@@ -360,84 +308,6 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
 	return response, nil
 }
 
-func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename string) (response *MoveFileResponse, err error) {
-	request := &MoveFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/mv.cgi",
-	}
-
-	response = &MoveFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't copy file")
-	}
-
-	return response, nil
-}
-
-func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
-	request := &CopyFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/cp.cgi",
-	}
-
-	response = &CopyFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't copy file")
-	}
-
-	return response, nil
-}
-
-func (f *Fs) renameFile(ctx context.Context, url string, newName string) (response *RenameFileResponse, err error) {
-	request := &RenameFileRequest{
-		URLs: []RenameFileURL{
-			{
-				URL:      url,
-				Filename: newName,
-			},
-		},
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/rename.cgi",
-	}
-
-	response = &RenameFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't rename file")
-	}
-
-	return response, nil
-}
-
 func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
 	// fs.Debugf(f, "Requesting Upload node")
 
@@ -450,7 +320,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 	response = &GetUploadNodeResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "didnt got an upload node")
@@ -493,7 +363,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if err != nil {
@@ -527,7 +397,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
 	response = &EndFileUploadResponse{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if err != nil {

backend/fichier/fichier.go

@@ -35,7 +35,9 @@ func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "fichier",
 		Description: "1Fichier",
-		NewFs:       NewFs,
+		Config: func(ctx context.Context, name string, config configmap.Mapper) {
+		},
+		NewFs: NewFs,
 		Options: []fs.Option{{
 			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
 			Name: "api_key",
@@ -44,18 +46,6 @@ func init() {
 			Name:     "shared_folder",
 			Required: false,
 			Advanced: true,
-		}, {
-			Help:       "If you want to download a shared file that is password protected, add this parameter",
-			Name:       "file_password",
-			Required:   false,
-			Advanced:   true,
-			IsPassword: true,
-		}, {
-			Help:       "If you want to list the files in a shared folder that is password protected, add this parameter",
-			Name:       "folder_password",
-			Required:   false,
-			Advanced:   true,
-			IsPassword: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -87,11 +77,9 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	APIKey         string               `config:"api_key"`
-	SharedFolder   string               `config:"shared_folder"`
-	FilePassword   string               `config:"file_password"`
-	FolderPassword string               `config:"folder_password"`
-	Enc            encoder.MultiEncoder `config:"encoding"`
+	APIKey       string               `config:"api_key"`
+	SharedFolder string               `config:"shared_folder"`
+	Enc          encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs is the interface a cloud storage system must provide
@@ -360,10 +348,8 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		return nil, err
 	}
 
-	if len(fileUploadResponse.Links) == 0 {
-		return nil, errors.New("upload response not found")
-	} else if len(fileUploadResponse.Links) > 1 {
-		fs.Debugf(remote, "Multiple upload responses found, using the first")
+	if len(fileUploadResponse.Links) != 1 {
+		return nil, errors.New("unexpected amount of files")
 	}
 
 	link := fileUploadResponse.Links[0]
@@ -377,6 +363,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		fs:     f,
 		remote: remote,
 		file: File{
+			ACL:         0,
 			CDN:         0,
 			Checksum:    link.Whirlpool,
 			ContentType: "",
@@ -429,109 +416,9 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return nil
 }
 
-// Move src to this remote using server side move operations.
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Find current directory ID
-	_, currentDirectoryID, err := f.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	// If it is in the correct directory, just rename it
-	var url string
-	if currentDirectoryID == directoryID {
-		resp, err := f.renameFile(ctx, srcObj.file.URL, leaf)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't rename file")
-		}
-		if resp.Status != "OK" {
-			return nil, errors.Errorf("couldn't rename file: %s", resp.Message)
-		}
-		url = resp.URLs[0].URL
-	} else {
-		folderID, err := strconv.Atoi(directoryID)
-		if err != nil {
-			return nil, err
-		}
-		resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't move file")
-		}
-		if resp.Status != "OK" {
-			return nil, errors.Errorf("couldn't move file: %s", resp.Message)
-		}
-		url = resp.URLs[0]
-	}
-
-	file, err := f.readFileInfo(ctx, url)
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// Copy src to this remote using server side move operations.
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf)
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't move file")
-	}
-	if resp.Status != "OK" {
-		return nil, errors.Errorf("couldn't move file: %s", resp.Message)
-	}
-
-	file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL)
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	o, err := f.NewObject(ctx, remote)
-	if err != nil {
-		return "", err
-	}
-	return o.(*Object).file.URL, nil
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
-	_ fs.Mover           = (*Fs)(nil)
-	_ fs.Copier          = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ dircache.DirCacher = (*Fs)(nil)
 )

backend/fichier/object.go

@@ -72,10 +72,6 @@ func (o *Object) SetModTime(context.Context, time.Time) error {
 	//return errors.New("setting modtime is not supported for 1fichier remotes")
 }
 
-func (o *Object) setMetaData(file File) {
-	o.file = file
-}
-
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
 	fs.FixRangeOption(options, o.file.Size)
@@ -94,7 +90,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.rest.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if err != nil {

backend/fichier/structs.go

@@ -1,10 +1,5 @@
 package fichier
 
-// FileInfoRequest is the request structure of the corresponding request
-type FileInfoRequest struct {
-	URL string `json:"url"`
-}
-
 // ListFolderRequest is the request structure of the corresponding request
 type ListFolderRequest struct {
 	FolderID int `json:"folder_id"`
@@ -19,7 +14,6 @@ type ListFilesRequest struct {
 type DownloadRequest struct {
 	URL    string `json:"url"`
 	Single int    `json:"single"`
-	Pass   string `json:"pass,omitempty"`
 }
 
 // RemoveFolderRequest is the request structure of the corresponding request
@@ -55,65 +49,6 @@ type MakeFolderResponse struct {
 	FolderID int    `json:"folder_id"`
 }
 
-// MoveFileRequest is the request structure of the corresponding request
-type MoveFileRequest struct {
-	URLs     []string `json:"urls"`
-	FolderID int      `json:"destination_folder_id"`
-	Rename   string   `json:"rename,omitempty"`
-}
-
-// MoveFileResponse is the response structure of the corresponding request
-type MoveFileResponse struct {
-	Status  string   `json:"status"`
-	Message string   `json:"message"`
-	URLs    []string `json:"urls"`
-}
-
-// CopyFileRequest is the request structure of the corresponding request
-type CopyFileRequest struct {
-	URLs     []string `json:"urls"`
-	FolderID int      `json:"folder_id"`
-	Rename   string   `json:"rename,omitempty"`
-}
-
-// CopyFileResponse is the response structure of the corresponding request
-type CopyFileResponse struct {
-	Status  string     `json:"status"`
-	Message string     `json:"message"`
-	Copied  int        `json:"copied"`
-	URLs    []FileCopy `json:"urls"`
-}
-
-// FileCopy is used in the the CopyFileResponse
-type FileCopy struct {
-	FromURL string `json:"from_url"`
-	ToURL   string `json:"to_url"`
-}
-
-// RenameFileURL is the data structure to rename a single file
-type RenameFileURL struct {
-	URL      string `json:"url"`
-	Filename string `json:"filename"`
-}
-
-// RenameFileRequest is the request structure of the corresponding request
-type RenameFileRequest struct {
-	URLs   []RenameFileURL `json:"urls"`
-	Pretty int             `json:"pretty"`
-}
-
-// RenameFileResponse is the response structure of the corresponding request
-type RenameFileResponse struct {
-	Status  string `json:"status"`
-	Message string `json:"message"`
-	Renamed int    `json:"renamed"`
-	URLs    []struct {
-		URL         string `json:"url"`
-		OldFilename string `json:"old_filename"`
-		NewFilename string `json:"new_filename"`
-	} `json:"urls"`
-}
-
 // GetUploadNodeResponse is the response structure of the corresponding request
 type GetUploadNodeResponse struct {
 	ID  string `json:"id"`
@@ -151,6 +86,7 @@ type EndFileUploadResponse struct {
 
 // File is the structure how 1Fichier returns a File
 type File struct {
+	ACL         int    `json:"acl"`
 	CDN         int    `json:"cdn"`
 	Checksum    string `json:"checksum"`
 	ContentType string `json:"content-type"`

backend/filefabric/api/types.go

@@ -5,7 +5,6 @@ package api
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"reflect"
 	"strings"
@@ -52,23 +51,6 @@ func (t Time) String() string {
 	return time.Time(t).UTC().Format(timeFormatParameters)
 }
 
-// Int represents an integer which can be represented in JSON as a
-// quoted integer or an integer.
-type Int int
-
-// MarshalJSON turns a Int into JSON
-func (i *Int) MarshalJSON() (out []byte, err error) {
-	return json.Marshal((*int)(i))
-}
-
-// UnmarshalJSON turns JSON into a Int
-func (i *Int) UnmarshalJSON(data []byte) error {
-	if len(data) >= 2 && data[0] == '"' && data[len(data)-1] == '"' {
-		data = data[1 : len(data)-1]
-	}
-	return json.Unmarshal(data, (*int)(i))
-}
-
 // Status return returned in all status responses
 type Status struct {
 	Code    string `json:"status"`
@@ -133,7 +115,7 @@ type GetFolderContentsResponse struct {
 	Total  int    `json:"total,string"`
 	Items  []Item `json:"filelist"`
 	Folder Item   `json:"folder"`
-	From   Int    `json:"from"`
+	From   int    `json:"from,string"`
 	//Count         int    `json:"count"`
 	Pid           string `json:"pid"`
 	RefreshResult Status `json:"refreshresult"`

backend/filefabric/filefabric.go

@@ -228,10 +228,7 @@ var retryStatusCodes = []struct {
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) {
 	if err != nil {
 		return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 	}
@@ -404,7 +401,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr
 		// Refresh the body each retry
 		opts.Body = strings.NewReader(data.Encode())
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, result)
-		return f.shouldRetry(ctx, resp, err, result)
+		return f.shouldRetry(resp, err, result)
 	})
 	if err != nil {
 		return resp, err
@@ -1280,7 +1277,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader)
-		return o.fs.shouldRetry(ctx, resp, err, nil)
+		return o.fs.shouldRetry(resp, err, nil)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to upload")

backend/ftp/ftp.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"crypto/tls"
 	"io"
-	"net"
 	"net/textproto"
 	"path"
 	"runtime"
@@ -21,8 +20,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
@@ -34,12 +31,6 @@ var (
 	currentUser = env.CurrentUser()
 )
 
-const (
-	minSleep      = 10 * time.Millisecond
-	maxSleep      = 2 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
-)
-
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -100,22 +91,6 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support",
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:    "idle_timeout",
-			Default: fs.Duration(60 * time.Second),
-			Help: `Max time before closing idle connections
-
-If no connections have been returned to the connection pool in the time
-given, rclone will empty the connection pool.
-
-Set to 0 to keep connections indefinitely.
-`,
-			Advanced: true,
-		}, {
-			Name:     "close_timeout",
-			Help:     "Maximum time to wait for a response to close.",
-			Default:  fs.Duration(60 * time.Second),
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -143,8 +118,6 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
-	IdleTimeout       fs.Duration          `config:"idle_timeout"`
-	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -161,10 +134,7 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
-	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
-	tlsConf  *tls.Config
-	pacer    *fs.Pacer // pacer for FTP connections
 }
 
 // Object describes an FTP file
@@ -241,48 +211,25 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-// shouldRetry returns a boolean as to whether this err deserve to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
-	}
-	return fserrors.ShouldRetry(err), err
-}
-
 // Open a new connection to the FTP server.
-func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
+func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
-
-	// Make ftp library dial with fshttp dialer optionally using TLS
-	dial := func(network, address string) (conn net.Conn, err error) {
-		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(f.ci.ConnectTimeout)}
+	if f.opt.TLS && f.opt.ExplicitTLS {
+		fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+	} else if f.opt.TLS {
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
 		}
-		return
-	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
-
-	if f.opt.TLS {
-		// Our dialer takes care of TLS but ftp library also needs tlsConf
-		// as a trigger for sending PSBZ and PROT options to server.
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
+		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
 	} else if f.opt.ExplicitTLS {
-		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
 		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -293,22 +240,18 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
-	err = f.pacer.Call(func() (bool, error) {
-		c, err = ftp.Dial(f.dialAddr, ftpConfig...)
-		if err != nil {
-			return shouldRetry(ctx, err)
-		}
-		err = c.Login(f.user, f.pass)
-		if err != nil {
-			_ = c.Quit()
-			return shouldRetry(ctx, err)
-		}
-		return false, nil
-	})
+	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
-		err = errors.Wrapf(err, "failed to make FTP connection to %q", f.dialAddr)
+		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
+		return nil, errors.Wrap(err, "ftpConnection Dial")
 	}
-	return c, err
+	err = c.Login(f.user, f.pass)
+	if err != nil {
+		_ = c.Quit()
+		fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
+		return nil, errors.Wrap(err, "ftpConnection Login")
+	}
+	return c, nil
 }
 
 // Get an FTP connection from the pool, or open a new one
@@ -365,32 +308,9 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
-	if f.opt.IdleTimeout > 0 {
-		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
-	}
 	f.poolMu.Unlock()
 }
 
-// Drain the pool of any connections
-func (f *Fs) drainPool(ctx context.Context) (err error) {
-	f.poolMu.Lock()
-	defer f.poolMu.Unlock()
-	if f.opt.IdleTimeout > 0 {
-		f.drain.Stop()
-	}
-	if len(f.pool) != 0 {
-		fs.Debugf(f, "closing %d unused connections", len(f.pool))
-	}
-	for i, c := range f.pool {
-		if cErr := c.Quit(); cErr != nil {
-			err = cErr
-		}
-		f.pool[i] = nil
-	}
-	f.pool = nil
-	return err
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
@@ -418,16 +338,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	if opt.TLS {
 		protocol = "ftps://"
 	}
-	if opt.TLS && opt.ExplicitTLS {
-		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
-	}
-	var tlsConfig *tls.Config
-	if opt.TLS || opt.ExplicitTLS {
-		tlsConfig = &tls.Config{
-			ServerName:         opt.Host,
-			InsecureSkipVerify: opt.SkipVerifyTLSCert,
-		}
-	}
 	u := protocol + path.Join(dialAddr+"/", root)
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
@@ -440,16 +350,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		pass:     pass,
 		dialAddr: dialAddr,
 		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
-		tlsConf:  tlsConfig,
-		pacer:    fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
-	// set the pool drainer timer going
-	if f.opt.IdleTimeout > 0 {
-		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
-	}
 	// Make a connection and pool it to return errors early
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
@@ -478,12 +382,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	return f, err
 }
 
-// Shutdown the backend, closing any background tasks and any
-// cached connections.
-func (f *Fs) Shutdown(ctx context.Context) error {
-	return f.drainPool(ctx)
-}
-
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
 	switch errX := err.(type) {
@@ -629,7 +527,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}()
 
 	// Wait for List for up to Timeout seconds
-	timer := time.NewTimer(f.ci.TimeoutOrInfinite())
+	timer := time.NewTimer(f.ci.Timeout)
 	select {
 	case listErr = <-errchan:
 		timer.Stop()
@@ -962,8 +860,8 @@ func (f *ftpReadCloser) Close() error {
 	go func() {
 		errchan <- f.rc.Close()
 	}()
-	// Wait for Close for up to 60 seconds by default
-	timer := time.NewTimer(time.Duration(f.f.opt.CloseTimeout))
+	// Wait for Close for up to 60 seconds
+	timer := time.NewTimer(60 * time.Second)
 	select {
 	case err = <-errchan:
 		timer.Stop()
@@ -1050,21 +948,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.Wrap(err, "Update")
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
-	// Ignore error 250 here - send by some servers
-	if err != nil {
-		switch errX := err.(type) {
-		case *textproto.Error:
-			switch errX.Code {
-			case ftp.StatusRequestedFileActionOK:
-				err = nil
-			}
-		}
-	}
 	if err != nil {
 		_ = c.Quit() // toss this connection to avoid sync errors
-		// recycle connection in advance to let remove() find free token
-		o.fs.putFtpConnection(nil, err)
 		remove()
+		o.fs.putFtpConnection(nil, err)
 		return errors.Wrap(err, "update stor")
 	}
 	o.fs.putFtpConnection(&c, nil)
@@ -1103,6 +990,5 @@ var (
 	_ fs.Mover       = &Fs{}
 	_ fs.DirMover    = &Fs{}
 	_ fs.PutStreamer = &Fs{}
-	_ fs.Shutdowner  = &Fs{}
 	_ fs.Object      = &Object{}
 )

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,9 +19,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -51,10 +51,10 @@ import (
 const (
 	rcloneClientID              = "202264815644.apps.googleusercontent.com"
 	rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"
-	timeFormat                  = time.RFC3339Nano
-	metaMtime                   = "mtime"                    // key to store mtime in metadata
-	metaMtimeGsutil             = "goog-reserved-file-mtime" // key used by GSUtil to store mtime in metadata
-	listChunks                  = 1000                       // chunk size to read directory listings
+	timeFormatIn                = time.RFC3339
+	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
+	metaMtime                   = "mtime" // key to store mtime under in metadata
+	listChunks                  = 1000    // chunk size to read directory listings
 	minSleep                    = 10 * time.Millisecond
 )
 
@@ -76,16 +76,17 @@ func init() {
 		Prefix:      "gcs",
 		Description: "Google Cloud Storage (this is not Google Drive)",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
 			if saFile != "" || saCreds != "" || anonymous == "true" {
-				return nil, nil
+				return
+			}
+			err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
 			}
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: storageConfig,
-			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "project_number",
@@ -328,10 +329,7 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func shouldRetry(ctx context.Context, err error) (again bool, errOut error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
 		if fserrors.ShouldRetry(err) {
@@ -457,7 +455,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err == nil {
 			newRoot := path.Dir(f.root)
@@ -523,7 +521,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
 			objects, err = list.Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			if gErr, ok := err.(*googleapi.Error); ok {
@@ -626,7 +624,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
 			buckets, err = listBuckets.Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return nil, err
@@ -752,7 +750,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err == nil {
 			// Bucket already exists
@@ -787,7 +785,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
 			_, err = insertBucket.Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 	}, nil)
 }
@@ -804,7 +802,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
 			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 	})
 }
@@ -850,7 +848,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	for {
 		err = f.pacer.Call(func() (bool, error) {
 			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return nil, err
@@ -921,7 +919,7 @@ func (o *Object) setMetaData(info *storage.Object) {
 	// read mtime out of metadata if available
 	mtimeString, ok := info.Metadata[metaMtime]
 	if ok {
-		modTime, err := time.Parse(timeFormat, mtimeString)
+		modTime, err := time.Parse(timeFormatIn, mtimeString)
 		if err == nil {
 			o.modTime = modTime
 			return
@@ -929,19 +927,8 @@ func (o *Object) setMetaData(info *storage.Object) {
 		fs.Debugf(o, "Failed to read mtime from metadata: %s", err)
 	}
 
-	// Fallback to GSUtil mtime
-	mtimeGsutilString, ok := info.Metadata[metaMtimeGsutil]
-	if ok {
-		unixTimeSec, err := strconv.ParseInt(mtimeGsutilString, 10, 64)
-		if err == nil {
-			o.modTime = time.Unix(unixTimeSec, 0)
-			return
-		}
-		fs.Debugf(o, "Failed to read GSUtil mtime from metadata: %s", err)
-	}
-
 	// Fallback to the Updated time
-	modTime, err := time.Parse(timeFormat, info.Updated)
+	modTime, err := time.Parse(timeFormatIn, info.Updated)
 	if err != nil {
 		fs.Logf(o, "Bad time decode: %v", err)
 	} else {
@@ -954,7 +941,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
@@ -998,8 +985,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 // Returns metadata for an object
 func metadataFromModTime(modTime time.Time) map[string]string {
 	metadata := make(map[string]string, 1)
-	metadata[metaMtime] = modTime.Format(timeFormat)
-	metadata[metaMtimeGsutil] = strconv.FormatInt(modTime.Unix(), 10)
+	metadata[metaMtime] = modTime.Format(timeFormatOut)
 	return metadata
 }
 
@@ -1011,11 +997,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 		return err
 	}
 	// Add the mtime to the existing metadata
+	mtime := modTime.Format(timeFormatOut)
 	if object.Metadata == nil {
 		object.Metadata = make(map[string]string, 1)
 	}
-	object.Metadata[metaMtime] = modTime.Format(timeFormat)
-	object.Metadata[metaMtimeGsutil] = strconv.FormatInt(modTime.Unix(), 10)
+	object.Metadata[metaMtime] = mtime
 	// Copy the object to itself to update the metadata
 	// Using PATCH requires too many permissions
 	bucket, bucketPath := o.split()
@@ -1026,7 +1012,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = copyObject.Context(ctx).Do()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -1042,10 +1028,11 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
+	req, err := http.NewRequest("GET", o.url, nil)
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	fs.FixRangeOption(options, o.bytes)
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response
@@ -1057,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1123,7 +1110,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
 		newObject, err = insertObject.Context(ctx).Do()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -1138,7 +1125,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	return err
 }

backend/googlephotos/googlephotos.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	golog "log"
 	"net/http"
 	"net/url"
 	"path"
@@ -53,7 +54,6 @@ const (
 	minSleep                    = 10 * time.Millisecond
 	scopeReadOnly               = "https://www.googleapis.com/auth/photoslibrary.readonly"
 	scopeReadWrite              = "https://www.googleapis.com/auth/photoslibrary"
-	scopeAccess                 = 2 // position of access scope in list
 )
 
 var (
@@ -62,7 +62,7 @@ var (
 		Scopes: []string{
 			"openid",
 			"profile",
-			scopeReadWrite, // this must be at position scopeAccess
+			scopeReadWrite,
 		},
 		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
@@ -78,36 +78,36 @@ func init() {
 		Prefix:      "gphotos",
 		Description: "Google Photos",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
 			// Parse config into Options struct
 			opt := new(Options)
 			err := configstruct.Set(m, opt)
 			if err != nil {
-				return nil, errors.Wrap(err, "couldn't parse config into struct")
+				fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
+				return
 			}
 
-			switch config.State {
-			case "":
-				// Fill in the scopes
-				if opt.ReadOnly {
-					oauthConfig.Scopes[scopeAccess] = scopeReadOnly
-				} else {
-					oauthConfig.Scopes[scopeAccess] = scopeReadWrite
-				}
-				return oauthutil.ConfigOut("warning", &oauthutil.Options{
-					OAuth2Config: oauthConfig,
-				})
-			case "warning":
-				// Warn the user as required by google photos integration
-				return fs.ConfigConfirm("warning_done", true, "config_warning", `Warning
-
-IMPORTANT: All media items uploaded to Google Photos with rclone
-are stored in full resolution at original quality.  These uploads
-will count towards storage in your Google Account.`)
-			case "warning_done":
-				return nil, nil
+			// Fill in the scopes
+			if opt.ReadOnly {
+				oauthConfig.Scopes[0] = scopeReadOnly
+			} else {
+				oauthConfig.Scopes[0] = scopeReadWrite
 			}
-			return nil, fmt.Errorf("unknown state %q", config.State)
+
+			// Do the oauth
+			err = oauthutil.Config(ctx, "google photos", name, m, oauthConfig, nil)
+			if err != nil {
+				golog.Fatalf("Failed to configure token: %v", err)
+			}
+
+			// Warn the user
+			fmt.Print(`
+*** IMPORTANT: All media items uploaded to Google Photos with rclone
+*** are stored in full resolution at original quality.  These uploads
+*** will count towards storage in your Google Account.
+
+`)
+
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "read_only",
@@ -240,10 +240,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -332,7 +329,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
 	var openIDconfig map[string]interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "couldn't read openID config")
@@ -361,7 +358,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't read user info")
@@ -392,7 +389,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 	var res interface{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't revoke token")
@@ -479,7 +476,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't list albums")
@@ -534,7 +531,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't list files")
@@ -678,7 +675,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create album")
@@ -813,7 +810,7 @@ func (o *Object) Size() int64 {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		fs.Debugf(o, "Reading size failed: %v", err)
@@ -864,7 +861,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 		var resp *http.Response
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "couldn't get media item")
@@ -941,7 +938,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -996,10 +993,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
 		if err != nil {
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		}
 		token, err = rest.ReadBody(resp)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't upload file")
@@ -1027,7 +1024,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.BatchCreateResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create media item")
@@ -1072,7 +1069,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't delete item from album")

backend/hdfs/fs.go

@@ -1,4 +1,3 @@
-//go:build !plan9
 // +build !plan9
 
 package hdfs

backend/hdfs/hdfs.go

@@ -1,4 +1,3 @@
-//go:build !plan9
 // +build !plan9
 
 package hdfs
@@ -38,7 +37,7 @@ func init() {
 			Help: `Kerberos service principal name for the namenode
 
 Enables KERBEROS authentication. Specifies the Service Principal Name
-(SERVICE/FQDN) for the namenode.`,
+(<SERVICE>/<FQDN>) for the namenode.`,
 			Required: false,
 			Examples: []fs.OptionExample{{
 				Value: "hdfs/namenode.hadoop.docker",

backend/hdfs/hdfs_test.go

@@ -1,6 +1,5 @@
 // Test HDFS filesystem interface
 
-//go:build !plan9
 // +build !plan9
 
 package hdfs_test

backend/hdfs/hdfs_unsupported.go

@@ -1,7 +1,6 @@
 // Build for hdfs for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9
 // +build plan9
 
 package hdfs

backend/hdfs/object.go

@@ -1,4 +1,3 @@
-//go:build !plan9
 // +build !plan9
 
 package hdfs
@@ -110,7 +109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	dirname := path.Dir(realpath)
 	fs.Debugf(o.fs, "update [%s]", realpath)
 
-	err := o.fs.client.MkdirAll(dirname, 0755)
+	err := o.fs.client.MkdirAll(dirname, 755)
 	if err != nil {
 		return err
 	}

backend/http/http.go

@@ -183,8 +183,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return http.ErrUseLastResponse
 		}
 		// check to see if points to a file
-		req, err := http.NewRequestWithContext(ctx, "HEAD", u.String(), nil)
+		req, err := http.NewRequest("HEAD", u.String(), nil)
 		if err == nil {
+			req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 			addHeaders(req, opt)
 			res, err := noRedir.Do(req)
 			err = statusError(res, err)
@@ -390,10 +391,11 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	// Do the request
-	req, err := http.NewRequestWithContext(ctx, "GET", URL, nil)
+	req, err := http.NewRequest("GET", URL, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "readDir failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	f.addHeaders(req)
 	res, err := f.httpClient.Do(req)
 	if err == nil {
@@ -545,10 +547,11 @@ func (o *Object) stat(ctx context.Context) error {
 		return nil
 	}
 	url := o.url()
-	req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
+	req, err := http.NewRequest("HEAD", url, nil)
 	if err != nil {
 		return errors.Wrap(err, "stat failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	o.fs.addHeaders(req)
 	res, err := o.fs.httpClient.Do(req)
 	if err == nil && res.StatusCode == http.StatusNotFound {
@@ -593,10 +596,11 @@ func (o *Object) Storable() bool {
 // Open a remote http file object for reading. Seek is supported
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	url := o.url()
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "Open failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 
 	// Add optional headers
 	for k, v := range fs.OpenOptionHeaders(options) {

backend/http/http_internal_test.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configfile"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/rest"
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	ts := httptest.NewServer(handler)
 
 	// Configure the remote
-	configfile.Install()
+	config.LoadConfig(context.Background())
 	// fs.Config.LogLevel = fs.LogLevelDebug
 	// fs.Config.DumpHeaders = true
 	// fs.Config.DumpBodies = true

backend/hubic/auth.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/ncw/swift/v2"
+	"github.com/ncw/swift"
 	"github.com/rclone/rclone/fs"
 )
 
@@ -24,7 +24,7 @@ func newAuth(f *Fs) *auth {
 // Request constructs an http.Request for authentication
 //
 // returns nil for not needed
-func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Request, err error) {
+func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {
 	const retries = 10
 	for try := 1; try <= retries; try++ {
 		err = a.f.getCredentials(context.TODO())
@@ -38,7 +38,7 @@ func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Reques
 }
 
 // Response parses the result of an http request
-func (a *auth) Response(ctx context.Context, resp *http.Response) error {
+func (a *auth) Response(resp *http.Response) error {
 	return nil
 }
 

backend/hubic/hubic.go

@@ -11,11 +11,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"strings"
 	"time"
 
-	swiftLib "github.com/ncw/swift/v2"
+	swiftLib "github.com/ncw/swift"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/swift"
 	"github.com/rclone/rclone/fs"
@@ -55,10 +56,11 @@ func init() {
 		Name:        "hubic",
 		Description: "Hubic",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
-			})
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+			err := oauthutil.Config(ctx, "hubic", name, m, oauthConfig, nil)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
 		},
 		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
 	})
@@ -108,10 +110,11 @@ func (f *Fs) String() string {
 //
 // The credentials are read into the Fs
 func (f *Fs) getCredentials(ctx context.Context) (err error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "https://api.hubic.com/1.0/account/credentials", nil)
+	req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
 	if err != nil {
 		return err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	resp, err := f.client.Do(req)
 	if err != nil {
 		return err
@@ -161,7 +164,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		Timeout:        10 * ci.Timeout,        // Use the timeouts in the transport
 		Transport:      fshttp.NewTransport(ctx),
 	}
-	err = c.Authenticate(ctx)
+	err = c.Authenticate()
 	if err != nil {
 		return nil, errors.Wrap(err, "error authenticating swift connection")
 	}

backend/jottacloud/jottacloud.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -48,29 +49,37 @@ const (
 	rootURL            = "https://jfs.jottacloud.com/jfs/"
 	apiURL             = "https://api.jottacloud.com/"
 	baseURL            = "https://www.jottacloud.com/"
+	defaultTokenURL    = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
 	cachePrefix        = "rclone-jcmd5-"
 	configDevice       = "device"
 	configMountpoint   = "mountpoint"
 	configTokenURL     = "tokenURL"
 	configClientID     = "client_id"
 	configClientSecret = "client_secret"
-	configUsername     = "username"
 	configVersion      = 1
 
-	defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
-	defaultClientID = "jottacli"
-
-	legacyTokenURL              = "https://api.jottacloud.com/auth/v1/token"
-	legacyRegisterURL           = "https://api.jottacloud.com/auth/v1/register"
-	legacyClientID              = "nibfk8biu12ju7hpqomr8b1e40"
-	legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
-	legacyConfigVersion         = 0
+	v1tokenURL              = "https://api.jottacloud.com/auth/v1/token"
+	v1registerURL           = "https://api.jottacloud.com/auth/v1/register"
+	v1ClientID              = "nibfk8biu12ju7hpqomr8b1e40"
+	v1EncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
+	v1configVersion         = 0
 
 	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
 	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
 	teliaCloudClientID = "desktop"
 )
 
+var (
+	// Description of how to auth for this app for a personal account
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
+		},
+		RedirectURL: oauthutil.RedirectLocalhostURL,
+	}
+)
+
 // Register with Fs
 func init() {
 	// needs to be done early so we can use oauth during config
@@ -78,7 +87,42 @@ func init() {
 		Name:        "jottacloud",
 		Description: "Jottacloud",
 		NewFs:       NewFs,
-		Config:      Config,
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+			refresh := false
+			if version, ok := m.Get("configVersion"); ok {
+				ver, err := strconv.Atoi(version)
+				if err != nil {
+					log.Fatalf("Failed to parse config version - corrupted config")
+				}
+				refresh = (ver != configVersion) && (ver != v1configVersion)
+			}
+
+			if refresh {
+				fmt.Printf("Config outdated - refreshing\n")
+			} else {
+				tokenString, ok := m.Get("token")
+				if ok && tokenString != "" {
+					fmt.Printf("Already have a token - refresh?\n")
+					if !config.Confirm(false) {
+						return
+					}
+				}
+			}
+
+			fmt.Printf("Choose authentication type:\n" +
+				"1: Standard authentication - use this if you're a normal Jottacloud user.\n" +
+				"2: Legacy authentication - this is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n" +
+				"3: Telia Cloud authentication - use this if you are using Telia Cloud.\n")
+
+			switch config.ChooseNumber("Your choice", 1, 3) {
+			case 1:
+				v2config(ctx, name, m)
+			case 2:
+				v1config(ctx, name, m)
+			case 3:
+				teliaCloudConfig(ctx, name, m)
+			}
+		},
 		Options: []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -99,11 +143,6 @@ func init() {
 			Help:     "Files bigger than this can be resumed if the upload fail's.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
-		}, {
-			Name:     "no_versions",
-			Help:     "Avoid server side versioning by deleting files and recreating files instead of overwriting them.",
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -118,183 +157,6 @@ func init() {
 	})
 }
 
-// Config runs the backend configuration protocol
-func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-	switch config.State {
-	case "":
-		return fs.ConfigChooseFixed("auth_type_done", "config_type", `Authentication type`, []fs.OptionExample{{
-			Value: "standard",
-			Help:  "Standard authentication - use this if you're a normal Jottacloud user.",
-		}, {
-			Value: "legacy",
-			Help:  "Legacy authentication - this is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
-		}, {
-			Value: "telia",
-			Help:  "Telia Cloud authentication - use this if you are using Telia Cloud.",
-		}})
-	case "auth_type_done":
-		// Jump to next state according to config chosen
-		return fs.ConfigGoto(config.Result)
-	case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
-		m.Set("configVersion", fmt.Sprint(configVersion))
-		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\n\nGenerate here: https://www.jottacloud.com/web/secure")
-	case "standard_token":
-		loginToken := config.Result
-		m.Set(configClientID, defaultClientID)
-		m.Set(configClientSecret, "")
-
-		srv := rest.NewClient(fshttp.NewClient(ctx))
-		token, tokenEndpoint, err := doTokenAuth(ctx, srv, loginToken)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get oauth token")
-		}
-		m.Set(configTokenURL, tokenEndpoint)
-		err = oauthutil.PutToken(name, m, &token, true)
-		if err != nil {
-			return nil, errors.Wrap(err, "error while saving token")
-		}
-		return fs.ConfigGoto("choose_device")
-	case "legacy": // configure a jottacloud backend using legacy authentication
-		m.Set("configVersion", fmt.Sprint(legacyConfigVersion))
-		return fs.ConfigConfirm("legacy_api", false, "config_machine_specific", `Do you want to create a machine specific API key?
-
-Rclone has it's own Jottacloud API KEY which works fine as long as one
-only uses rclone on a single machine. When you want to use rclone with
-this account on more than one machine it's recommended to create a
-machine specific API key. These keys can NOT be shared between
-machines.`)
-	case "legacy_api":
-		srv := rest.NewClient(fshttp.NewClient(ctx))
-		if config.Result == "true" {
-			deviceRegistration, err := registerDevice(ctx, srv)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to register device")
-			}
-			m.Set(configClientID, deviceRegistration.ClientID)
-			m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
-			fs.Debugf(nil, "Got clientID %q and clientSecret %q", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
-		}
-		return fs.ConfigInput("legacy_username", "config_username", "Username (e-mail address)")
-	case "legacy_username":
-		m.Set(configUsername, config.Result)
-		return fs.ConfigPassword("legacy_password", "config_password", "Password (only used in setup, will not be stored)")
-	case "legacy_password":
-		m.Set("password", config.Result)
-		m.Set("auth_code", "")
-		return fs.ConfigGoto("legacy_do_auth")
-	case "legacy_auth_code":
-		authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
-		m.Set("auth_code", authCode)
-		return fs.ConfigGoto("legacy_do_auth")
-	case "legacy_do_auth":
-		username, _ := m.Get(configUsername)
-		password, _ := m.Get("password")
-		password = obscure.MustReveal(password)
-		authCode, _ := m.Get("auth_code")
-
-		srv := rest.NewClient(fshttp.NewClient(ctx))
-		clientID, ok := m.Get(configClientID)
-		if !ok {
-			clientID = legacyClientID
-		}
-		clientSecret, ok := m.Get(configClientSecret)
-		if !ok {
-			clientSecret = legacyEncryptedClientSecret
-		}
-
-		oauthConfig := &oauth2.Config{
-			Endpoint: oauth2.Endpoint{
-				AuthURL: legacyTokenURL,
-			},
-			ClientID:     clientID,
-			ClientSecret: obscure.MustReveal(clientSecret),
-		}
-		token, err := doLegacyAuth(ctx, srv, oauthConfig, username, password, authCode)
-		if err == errAuthCodeRequired {
-			return fs.ConfigInput("legacy_auth_code", "config_auth_code", "Verification Code\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
-		}
-		m.Set("password", "")
-		m.Set("auth_code", "")
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get oauth token")
-		}
-		err = oauthutil.PutToken(name, m, &token, true)
-		if err != nil {
-			return nil, errors.Wrap(err, "error while saving token")
-		}
-		return fs.ConfigGoto("choose_device")
-	case "telia": // telia cloud config
-		m.Set("configVersion", fmt.Sprint(configVersion))
-		m.Set(configClientID, teliaCloudClientID)
-		m.Set(configTokenURL, teliaCloudTokenURL)
-		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
-					AuthURL:  teliaCloudAuthURL,
-					TokenURL: teliaCloudTokenURL,
-				},
-				ClientID:    teliaCloudClientID,
-				Scopes:      []string{"openid", "jotta-default", "offline_access"},
-				RedirectURL: oauthutil.RedirectLocalhostURL,
-			},
-		})
-	case "choose_device":
-		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
-	case "choose_device_query":
-		if config.Result != "true" {
-			m.Set(configDevice, "")
-			m.Set(configMountpoint, "")
-			return fs.ConfigGoto("end")
-		}
-		oAuthClient, _, err := getOAuthClient(ctx, name, m)
-		if err != nil {
-			return nil, err
-		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
-		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
-
-		cust, err := getCustomerInfo(ctx, apiSrv)
-		if err != nil {
-			return nil, err
-		}
-		m.Set(configUsername, cust.Username)
-
-		acc, err := getDriveInfo(ctx, srv, cust.Username)
-		if err != nil {
-			return nil, err
-		}
-		return fs.ConfigChoose("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) {
-			return acc.Devices[i].Name, ""
-		})
-	case "choose_device_result":
-		device := config.Result
-		m.Set(configDevice, device)
-
-		oAuthClient, _, err := getOAuthClient(ctx, name, m)
-		if err != nil {
-			return nil, err
-		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
-
-		username, _ := m.Get(configUsername)
-		dev, err := getDeviceInfo(ctx, srv, path.Join(username, device))
-		if err != nil {
-			return nil, err
-		}
-		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) {
-			return dev.MountPoints[i].Name, ""
-		})
-	case "choose_device_mountpoint":
-		mountpoint := config.Result
-		m.Set(configMountpoint, mountpoint)
-		return fs.ConfigGoto("end")
-	case "end":
-		// All the config flows end up here in case we need to carry on with something
-		return nil, nil
-	}
-	return nil, fmt.Errorf("unknown state %q", config.State)
-}
-
 // Options defines the configuration for this backend
 type Options struct {
 	Device             string               `config:"device"`
@@ -302,7 +164,6 @@ type Options struct {
 	MD5MemoryThreshold fs.SizeSuffix        `config:"md5_memory_limit"`
 	TrashedOnly        bool                 `config:"trashed_only"`
 	HardDelete         bool                 `config:"hard_delete"`
-	NoVersions         bool                 `config:"no_versions"`
 	UploadThreshold    fs.SizeSuffix        `config:"upload_resume_limit"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 }
@@ -356,21 +217,10 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// joinPath joins two path/url elements
-//
-// Does not perform clean on the result like path.Join does,
-// which breaks urls by changing prefix "https://" into "https:/".
-func joinPath(base string, rel string) string {
-	if rel == "" {
-		return base
-	}
-	if strings.HasSuffix(base, "/") {
-		return base + strings.TrimPrefix(rel, "/")
-	}
-	if strings.HasPrefix(rel, "/") {
-		return strings.TrimSuffix(base, "/") + rel
-	}
-	return base + "/" + rel
+// parsePath parses a box 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
 }
 
 // retryErrorCodes is a slice of error codes that we will retry
@@ -385,13 +235,114 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
+func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
+	teliaCloudOauthConfig := &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  teliaCloudAuthURL,
+			TokenURL: teliaCloudTokenURL,
+		},
+		ClientID:    teliaCloudClientID,
+		Scopes:      []string{"openid", "jotta-default", "offline_access"},
+		RedirectURL: oauthutil.RedirectLocalhostURL,
+	}
+
+	err := oauthutil.Config(ctx, "jottacloud", name, m, teliaCloudOauthConfig, nil)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+		return
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, teliaCloudOauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(configVersion))
+	m.Set(configClientID, teliaCloudClientID)
+	m.Set(configTokenURL, teliaCloudTokenURL)
+}
+
+// v1config configure a jottacloud backend using legacy authentication
+func v1config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(ctx))
+
+	fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+	if config.Confirm(false) {
+		deviceRegistration, err := registerDevice(ctx, srv)
+		if err != nil {
+			log.Fatalf("Failed to register device: %v", err)
+		}
+
+		m.Set(configClientID, deviceRegistration.ClientID)
+		m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+		fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+	}
+
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = v1ClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = v1EncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+	oauthConfig.Endpoint.AuthURL = v1tokenURL
+	oauthConfig.Endpoint.TokenURL = v1tokenURL
+
+	fmt.Printf("Username> ")
+	username := config.ReadLine()
+	password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
+
+	token, err := doAuthV1(ctx, srv, username, password)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(v1configVersion))
+}
+
 // registerDevice register a new device for use with the jottacloud API
 func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
 	// random generator to generate random device names
@@ -410,7 +361,7 @@ func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegis
 
 	opts := rest.Opts{
 		Method:       "POST",
-		RootURL:      legacyRegisterURL,
+		RootURL:      v1registerURL,
 		ContentType:  "application/x-www-form-urlencoded",
 		ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
 		Parameters:   values,
@@ -421,13 +372,8 @@ func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegis
 	return deviceRegistration, err
 }
 
-var errAuthCodeRequired = errors.New("auth code required")
-
-// doLegacyAuth runs the actual token request for V1 authentication
-//
-// Call this first with blank authCode. If errAuthCodeRequired is
-// returned then call it again with an authCode
-func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Config, username, password, authCode string) (token oauth2.Token, err error) {
+// doAuthV1 runs the actual token request for V1 authentication
+func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
 	// prepare out token request with username and password
 	values := url.Values{}
 	values.Set("grant_type", "PASSWORD")
@@ -441,19 +387,22 @@ func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Con
 		ContentType: "application/x-www-form-urlencoded",
 		Parameters:  values,
 	}
-	if authCode != "" {
-		opts.ExtraHeaders = make(map[string]string)
-		opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-	}
 
 	// do the first request
 	var jsonToken api.TokenJSON
 	resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
-	if err != nil && authCode == "" {
+	if err != nil {
 		// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
 		if resp != nil {
 			if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-				return token, errAuthCodeRequired
+				fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
+				fmt.Printf("Enter verification code> ")
+				authCode := config.ReadLine()
+
+				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
+				opts.ExtraHeaders = make(map[string]string)
+				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
+				_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
 			}
 		}
 	}
@@ -465,11 +414,51 @@ func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Con
 	return token, err
 }
 
-// doTokenAuth runs the actual token request for V2 authentication
-func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, err error) {
+// v2config configure a jottacloud backend using the modern JottaCli token based authentication
+func v2config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(ctx))
+
+	fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
+	fmt.Printf("Login Token> ")
+	loginToken := config.ReadLine()
+
+	m.Set(configClientID, "jottacli")
+	m.Set(configClientSecret, "")
+
+	token, err := doAuthV2(ctx, srv, loginToken, m)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(configVersion))
+}
+
+// doAuthV2 runs the actual token request for V2 authentication
+func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
-		return token, "", err
+		return token, err
 	}
 
 	// decode login token
@@ -477,7 +466,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
 	decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
 	err = decoder.Decode(&loginToken)
 	if err != nil {
-		return token, "", err
+		return token, err
 	}
 
 	// retrieve endpoint urls
@@ -486,14 +475,19 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
 		RootURL: loginToken.WellKnownLink,
 	}
 	var wellKnown api.WellKnown
-	_, err = apiSrv.CallJSON(ctx, &opts, nil, &wellKnown)
+	_, err = srv.CallJSON(ctx, &opts, nil, &wellKnown)
 	if err != nil {
-		return token, "", err
+		return token, err
 	}
 
+	// save the tokenurl
+	oauthConfig.Endpoint.AuthURL = wellKnown.TokenEndpoint
+	oauthConfig.Endpoint.TokenURL = wellKnown.TokenEndpoint
+	m.Set(configTokenURL, wellKnown.TokenEndpoint)
+
 	// prepare out token request with username and password
 	values := url.Values{}
-	values.Set("client_id", defaultClientID)
+	values.Set("client_id", "jottacli")
 	values.Set("grant_type", "password")
 	values.Set("password", loginToken.AuthToken)
 	values.Set("scope", "offline_access+openid")
@@ -501,33 +495,68 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
 	values.Encode()
 	opts = rest.Opts{
 		Method:      "POST",
-		RootURL:     wellKnown.TokenEndpoint,
+		RootURL:     oauthConfig.Endpoint.AuthURL,
 		ContentType: "application/x-www-form-urlencoded",
 		Body:        strings.NewReader(values.Encode()),
 	}
 
 	// do the first request
 	var jsonToken api.TokenJSON
-	_, err = apiSrv.CallJSON(ctx, &opts, nil, &jsonToken)
+	_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
 	if err != nil {
-		return token, "", err
+		return token, err
 	}
 
 	token.AccessToken = jsonToken.AccessToken
 	token.RefreshToken = jsonToken.RefreshToken
 	token.TokenType = jsonToken.TokenType
 	token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
-	return token, wellKnown.TokenEndpoint, err
+	return token, err
+}
+
+// setupMountpoint sets up a custom device and mountpoint if desired by the user
+func setupMountpoint(ctx context.Context, srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) {
+	cust, err := getCustomerInfo(ctx, apiSrv)
+	if err != nil {
+		return "", "", err
+	}
+
+	acc, err := getDriveInfo(ctx, srv, cust.Username)
+	if err != nil {
+		return "", "", err
+	}
+	var deviceNames []string
+	for i := range acc.Devices {
+		deviceNames = append(deviceNames, acc.Devices[i].Name)
+	}
+	fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
+	device = config.Choose("Devices", deviceNames, nil, false)
+
+	dev, err := getDeviceInfo(ctx, srv, path.Join(cust.Username, device))
+	if err != nil {
+		return "", "", err
+	}
+	if len(dev.MountPoints) == 0 {
+		return "", "", errors.New("no mountpoints for selected device")
+	}
+	var mountpointNames []string
+	for i := range dev.MountPoints {
+		mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
+	}
+	fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
+	mountpoint = config.Choose("Mountpoints", mountpointNames, nil, false)
+
+	return device, mountpoint, err
 }
 
 // getCustomerInfo queries general information about the account
-func getCustomerInfo(ctx context.Context, apiSrv *rest.Client) (info *api.CustomerInfo, err error) {
+func getCustomerInfo(ctx context.Context, srv *rest.Client) (info *api.CustomerInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "account/v1/customer",
 	}
 
-	_, err = apiSrv.CallJSON(ctx, &opts, nil, &info)
+	_, err = srv.CallJSON(ctx, &opts, nil, &info)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get customer info")
 	}
@@ -586,7 +615,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -644,7 +673,7 @@ func (f *Fs) filePath(file string) string {
 // This filter catches all refresh requests, reads the body,
 // changes the case and then sends it on
 func grantTypeFilter(req *http.Request) {
-	if legacyTokenURL == req.URL.String() {
+	if v1tokenURL == req.URL.String() {
 		// read the entire body
 		refreshBody, err := ioutil.ReadAll(req.Body)
 		if err != nil {
@@ -660,50 +689,53 @@ func grantTypeFilter(req *http.Request) {
 	}
 }
 
-func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuthClient *http.Client, ts *oauthutil.TokenSource, err error) {
+// NewFs constructs an Fs from the path, container:path
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
 	// Check config version
 	var ver int
 	version, ok := m.Get("configVersion")
 	if ok {
 		ver, err = strconv.Atoi(version)
 		if err != nil {
-			return nil, nil, errors.New("Failed to parse config version")
+			return nil, errors.New("Failed to parse config version")
 		}
-		ok = (ver == configVersion) || (ver == legacyConfigVersion)
+		ok = (ver == configVersion) || (ver == v1configVersion)
 	}
 	if !ok {
-		return nil, nil, errors.New("Outdated config - please reconfigure this backend")
+		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
 	baseClient := fshttp.NewClient(ctx)
-	oauthConfig := &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  defaultTokenURL,
-			TokenURL: defaultTokenURL,
-		},
-	}
+
 	if ver == configVersion {
-		oauthConfig.ClientID = defaultClientID
+		oauthConfig.ClientID = "jottacli"
 		// if custom endpoints are set use them else stick with defaults
 		if tokenURL, ok := m.Get(configTokenURL); ok {
 			oauthConfig.Endpoint.TokenURL = tokenURL
 			// jottacloud is weird. we need to use the tokenURL as authURL
 			oauthConfig.Endpoint.AuthURL = tokenURL
 		}
-	} else if ver == legacyConfigVersion {
+	} else if ver == v1configVersion {
 		clientID, ok := m.Get(configClientID)
 		if !ok {
-			clientID = legacyClientID
+			clientID = v1ClientID
 		}
 		clientSecret, ok := m.Get(configClientSecret)
 		if !ok {
-			clientSecret = legacyEncryptedClientSecret
+			clientSecret = v1EncryptedClientSecret
 		}
 		oauthConfig.ClientID = clientID
 		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-		oauthConfig.Endpoint.TokenURL = legacyTokenURL
-		oauthConfig.Endpoint.AuthURL = legacyTokenURL
+		oauthConfig.Endpoint.TokenURL = v1tokenURL
+		oauthConfig.Endpoint.AuthURL = v1tokenURL
 
 		// add the request filter to fix token refresh
 		if do, ok := baseClient.Transport.(interface {
@@ -716,29 +748,13 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	}
 
 	// Create OAuth Client
-	oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
+	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
-	}
-	return oAuthClient, ts, nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	// Parse config into Options struct
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	oAuthClient, ts, err := getOAuthClient(ctx, name, m)
-	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
 	rootIsDir := strings.HasSuffix(root, "/")
-	root = strings.Trim(root, "/")
+	root = parsePath(root)
 
 	f := &Fs{
 		name:   name,
@@ -838,7 +854,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -867,7 +883,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var result api.JottaFolder
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if err != nil {
@@ -979,7 +995,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1085,7 +1101,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "couldn't purge directory")
@@ -1124,7 +1140,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1252,7 +1268,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var result api.JottaFile
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	if apiErr, ok := err.(*api.Error); ok {
@@ -1276,7 +1292,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	if result.PublicSharePath == "" {
 		return "", errors.New("couldn't create public link - no link path received")
 	}
-	return joinPath(baseURL, result.PublicSharePath), nil
+	link = path.Join(baseURL, result.PublicSharePath)
+	return link, nil
 }
 
 // About gets quota information
@@ -1429,7 +1446,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1500,20 +1517,6 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	if o.fs.opt.NoVersions {
-		err := o.readMetaData(ctx, false)
-		if err == nil {
-			// if the object exists delete it
-			err = o.remove(ctx, true)
-			if err != nil {
-				return errors.Wrap(err, "failed to remove old object")
-			}
-		}
-		// if the object does not exist we can just continue but if the error is something different we should report that
-		if err != fs.ErrorObjectNotFound {
-			return err
-		}
-	}
 	o.fs.tokenRenewer.Start()
 	defer o.fs.tokenRenewer.Stop()
 	size := src.Size()
@@ -1556,7 +1559,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var response api.AllocateFileResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1604,7 +1607,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return nil
 }
 
-func (o *Object) remove(ctx context.Context, hard bool) error {
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
 	opts := rest.Opts{
 		Method:     "POST",
 		Path:       o.filePath(),
@@ -1612,7 +1616,7 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 		NoResponse: true,
 	}
 
-	if hard {
+	if o.fs.opt.HardDelete {
 		opts.Parameters.Set("rm", "true")
 	} else {
 		opts.Parameters.Set("dl", "true")
@@ -1620,15 +1624,10 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	return o.remove(ctx, o.fs.opt.HardDelete)
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs           = (*Fs)(nil)

backend/koofr/koofr.go

@@ -534,7 +534,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return nil
 }
 
-// About reports space usage (with a MiB precision)
+// About reports space usage (with a MB precision)
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	mount, err := f.client.MountsDetails(f.mountID)
 	if err != nil {

backend/local/about_unix.go

@@ -1,4 +1,3 @@
-//go:build darwin || dragonfly || freebsd || linux
 // +build darwin dragonfly freebsd linux
 
 package local

backend/local/about_windows.go

@@ -1,4 +1,3 @@
-//go:build windows
 // +build windows
 
 package local

backend/local/encode_darwin.go

@@ -1,5 +1,4 @@
-//go:build darwin
-// +build darwin
+//+build darwin
 
 package local
 

backend/local/encode_other.go

@@ -1,5 +1,4 @@
-//go:build !windows && !darwin
-// +build !windows,!darwin
+//+build !windows,!darwin
 
 package local
 

backend/local/encode_windows.go

@@ -1,5 +1,4 @@
-//go:build windows
-// +build windows
+//+build windows
 
 package local
 

backend/local/fadvise_other.go

@@ -1,5 +1,4 @@
-//go:build !linux
-// +build !linux
+//+build !linux
 
 package local
 

backend/local/fadvise_unix.go

@@ -1,5 +1,4 @@
-//go:build linux
-// +build linux
+//+build linux
 
 package local
 

backend/local/lchtimes.go

@@ -1,4 +1,3 @@
-//go:build windows || plan9 || js
 // +build windows plan9 js
 
 package local

backend/local/lchtimes_unix.go

@@ -1,4 +1,3 @@
-//go:build !windows && !plan9 && !js
 // +build !windows,!plan9,!js
 
 package local

backend/local/local.go

@@ -27,7 +27,6 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
-	"golang.org/x/text/unicode/norm"
 )
 
 // Constants
@@ -43,9 +42,8 @@ func init() {
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
-			Name:     "nounc",
-			Help:     "Disable UNC (long path names) conversion on Windows",
-			Advanced: runtime.GOOS != "windows",
+			Name: "nounc",
+			Help: "Disable UNC (long path names) conversion on Windows",
 			Examples: []fs.OptionExample{{
 				Value: "true",
 				Help:  "Disables long file names",
@@ -74,34 +72,25 @@ points, as you explicitly acknowledge that they should be skipped.`,
 			Advanced: true,
 		}, {
 			Name: "zero_size_links",
-			Help: `Assume the Stat size of links is zero (and read them instead) (Deprecated)
+			Help: `Assume the Stat size of links is zero (and read them instead)
 
-Rclone used to use the Stat size of links as the link size, but this fails in quite a few places
+On some virtual filesystems (such ash LucidLink), reading a link size via a Stat call always returns 0.
+However, on unix it reads as the length of the text in the link. This may cause errors like this when
+syncing:
 
-- Windows
-- On some virtual filesystems (such ash LucidLink)
-- Android
+    Failed to copy: corrupted on transfer: sizes differ 0 vs 13
 
-So rclone now always reads the link
-`,
+Setting this flag causes rclone to read the link and use that as the size of the link
+instead of 0 which in most cases fixes the problem.`,
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name: "unicode_normalization",
-			Help: `Apply unicode NFC normalization to paths and filenames
+			Name: "no_unicode_normalization",
+			Help: `Don't apply unicode normalization to paths and filenames (Deprecated)
 
-This flag can be used to normalize file names into unicode NFC form
-that are read from the local filesystem.
-
-Rclone does not normally touch the encoding of file names it reads from
-the file system.
-
-This can be useful when using macOS as it normally provides decomposed (NFD)
-unicode which in some language (eg Korean) doesn't display properly on
-some OSes.
-
-Note that rclone compares filenames with unicode normalization in the sync
-routine so this flag shouldn't normally be used.`,
+This flag is deprecated now.  Rclone no longer normalizes unicode file
+names, but it compares them with unicode normalization in the sync
+routine instead.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -159,17 +148,6 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "no_preallocate",
-			Help: `Disable preallocation of disk space for transferred files
-
-Preallocation of disk space helps prevent filesystem fragmentation.
-However, some virtual filesystem layers (such as Google Drive File
-Stream) may incorrectly set the actual file size equal to the
-preallocated space, causing checksum and file size checks to fail.
-Use this flag to disable preallocation.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name: "no_sparse",
 			Help: `Disable sparse files for multi-thread downloads
@@ -206,13 +184,13 @@ type Options struct {
 	FollowSymlinks    bool                 `config:"copy_links"`
 	TranslateSymlinks bool                 `config:"links"`
 	SkipSymlinks      bool                 `config:"skip_links"`
-	UTFNorm           bool                 `config:"unicode_normalization"`
+	ZeroSizeLinks     bool                 `config:"zero_size_links"`
+	NoUTFNorm         bool                 `config:"no_unicode_normalization"`
 	NoCheckUpdated    bool                 `config:"no_check_updated"`
 	NoUNC             bool                 `config:"nounc"`
 	OneFileSystem     bool                 `config:"one_file_system"`
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
-	NoPreAllocate     bool                 `config:"no_preallocate"`
 	NoSparse          bool                 `config:"no_sparse"`
 	NoSetModTime      bool                 `config:"no_set_modtime"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
@@ -265,6 +243,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, errLinksAndCopyLinks
 	}
 
+	if opt.NoUTFNorm {
+		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
+	}
+
 	f := &Fs{
 		name:   name,
 		opt:    *opt,
@@ -467,10 +449,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				for _, name := range names {
 					namepath := filepath.Join(fsDirPath, name)
 					fi, fierr := os.Lstat(namepath)
-					if os.IsNotExist(fierr) {
-						// skip entry removed by a concurrent goroutine
-						continue
-					}
 					if fierr != nil {
 						err = errors.Wrapf(err, "failed to read directory %q", namepath)
 						fs.Errorf(dir, "%v", fierr)
@@ -531,9 +509,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 }
 
 func (f *Fs) cleanRemote(dir, filename string) (remote string) {
-	if f.opt.UTFNorm {
-		filename = norm.NFC.String(filename)
-	}
 	remote = path.Join(dir, f.opt.Enc.ToStandardName(filename))
 
 	if !utf8.ValidString(filename) {
@@ -1152,16 +1127,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				return err
 			}
 		}
-		if !o.fs.opt.NoPreAllocate {
-			// Pre-allocate the file for performance reasons
-			err = file.PreAllocate(src.Size(), f)
-			if err != nil {
-				fs.Debugf(o, "Failed to pre-allocate: %v", err)
-				if err == file.ErrDiskFull {
-					_ = f.Close()
-					return err
-				}
-			}
+		// Pre-allocate the file for performance reasons
+		err = file.PreAllocate(src.Size(), f)
+		if err != nil {
+			fs.Debugf(o, "Failed to pre-allocate: %v", err)
 		}
 		out = f
 	} else {
@@ -1248,11 +1217,9 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 		return nil, err
 	}
 	// Pre-allocate the file for performance reasons
-	if !f.opt.NoPreAllocate {
-		err = file.PreAllocate(size, out)
-		if err != nil {
-			fs.Debugf(o, "Failed to pre-allocate: %v", err)
-		}
+	err = file.PreAllocate(size, out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
 	if !f.opt.NoSparse && file.SetSparseImplemented {
 		sparseWarning.Do(func() {
@@ -1279,13 +1246,9 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	o.modTime = info.ModTime()
 	o.mode = info.Mode()
 	o.fs.objectMetaMu.Unlock()
-	// Read the size of the link.
-	//
-	// The value in info.Size() is not always correct
-	// - Windows links read as 0 size
-	// - Some virtual filesystems (such ash LucidLink) links read as 0 size
-	// - Android - some versions the links are larger than readlink suggests
-	if o.translatedLink {
+	// On Windows links read as 0 size so set the correct size here
+	// Optionally, users can turn this feature on with the zero_size_links flag
+	if (runtime.GOOS == "windows" || o.fs.opt.ZeroSizeLinks) && o.translatedLink {
 		linkdst, err := os.Readlink(o.path)
 		if err != nil {
 			fs.Errorf(o, "Failed to read link size: %v", err)

backend/local/read_device_other.go

@@ -1,6 +1,5 @@
 // Device reading functions
 
-//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris
 // +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
 
 package local

backend/local/read_device_unix.go

@@ -1,6 +1,5 @@
 // Device reading functions
 
-//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
 // +build darwin dragonfly freebsd linux netbsd openbsd solaris
 
 package local

backend/local/remove_other.go

@@ -1,5 +1,4 @@
-//go:build !windows
-// +build !windows
+//+build !windows
 
 package local
 

backend/local/remove_windows.go

@@ -1,5 +1,4 @@
-//go:build windows
-// +build windows
+//+build windows
 
 package local
 

backend/local/symlink.go

@@ -1,4 +1,3 @@
-//go:build !windows && !plan9 && !js
 // +build !windows,!plan9,!js
 
 package local

backend/local/symlink_other.go

@@ -1,4 +1,3 @@
-//go:build windows || plan9 || js
 // +build windows plan9 js
 
 package local

backend/mailru/api/helpers.go

@@ -6,8 +6,8 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"io"
+	"log"
 	"time"
 
 	"github.com/pkg/errors"
@@ -48,7 +48,7 @@ func (w *BinWriter) Reader() io.Reader {
 // WritePu16 writes a short as unsigned varint
 func (w *BinWriter) WritePu16(val int) {
 	if val < 0 || val > 65535 {
-		panic(fmt.Sprintf("Invalid UInt16 %v", val))
+		log.Fatalf("Invalid UInt16 %v", val)
 	}
 	w.WritePu64(int64(val))
 }
@@ -56,7 +56,7 @@ func (w *BinWriter) WritePu16(val int) {
 // WritePu32 writes a signed long as unsigned varint
 func (w *BinWriter) WritePu32(val int64) {
 	if val < 0 || val > 4294967295 {
-		panic(fmt.Sprintf("Invalid UInt32 %v", val))
+		log.Fatalf("Invalid UInt32 %v", val)
 	}
 	w.WritePu64(val)
 }
@@ -64,7 +64,7 @@ func (w *BinWriter) WritePu32(val int64) {
 // WritePu64 writes an unsigned (actually, signed) long as unsigned varint
 func (w *BinWriter) WritePu64(val int64) {
 	if val < 0 {
-		panic(fmt.Sprintf("Invalid UInt64 %v", val))
+		log.Fatalf("Invalid UInt64 %v", val)
 	}
 	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
 }
@@ -123,7 +123,7 @@ func (r *BinReader) check(err error) bool {
 		r.err = err
 	}
 	if err != io.EOF {
-		panic(fmt.Sprintf("Error parsing response: %v", err))
+		log.Fatalf("Error parsing response: %v", err)
 	}
 	return false
 }

backend/mailru/mailru.go

@@ -80,7 +80,7 @@ var oauthConfig = &oauth2.Config{
 
 // Register with Fs
 func init() {
-	MrHashType = hash.RegisterHash("mailru", "MailruHash", 40, mrhash.New)
+	MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "mailru",
 		Description: "Mail.ru Cloud",
@@ -234,10 +234,7 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this response and err
 // deserve to be retried. It returns the err as a convenience.
 // Retries password authorization (once) in a special case of access denied.
-func shouldRetry(ctx context.Context, res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
 	if res != nil && res.StatusCode == 403 && f.opt.Password != "" && !f.passFailed {
 		reAuthErr := f.reAuthorize(opts, err)
 		return reAuthErr == nil, err // return an original error
@@ -603,7 +600,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 	var info api.ItemInfoResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -739,7 +736,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int)
 	)
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -803,7 +800,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1076,7 +1073,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1219,7 +1216,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
 	var response api.GenericResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 
 	switch {
@@ -1291,7 +1288,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var response api.GenericBodyResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 
 	if err != nil {
@@ -1395,7 +1392,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
 	var res *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		res, err = f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -1486,7 +1483,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var response api.GenericBodyResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 
 	if err == nil && response.Body != "" {
@@ -1527,7 +1524,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	var response api.CleanupResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 	if err != nil {
 		return err
@@ -1560,7 +1557,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var info api.UserInfoResponse
 	err = f.pacer.Call(func() (bool, error) {
 		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, res, err, f, &opts)
+		return shouldRetry(res, err, f, &opts)
 	})
 	if err != nil {
 		return nil, err
@@ -2079,7 +2076,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	var res *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err, o.fs, &opts)
+		return shouldRetry(res, err, o.fs, &opts)
 	})
 	if err != nil {
 		closeBody(res)
@@ -2175,7 +2172,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		opts.RootURL = server
 		res, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err, o.fs, &opts)
+		return shouldRetry(res, err, o.fs, &opts)
 	})
 	if err != nil {
 		if res != nil && res.Body != nil {

backend/mega/mega.go

@@ -30,7 +30,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -159,10 +158,7 @@ func parsePath(path string) (root string) {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(err error) (bool, error) {
 	// Let the mega library handle the low level retries
 	return false, err
 	/*
@@ -175,8 +171,8 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 }
 
 // readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForPath(ctx context.Context, remote string) (info *mega.Node, err error) {
-	rootNode, err := f.findRoot(ctx, false)
+func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) {
+	rootNode, err := f.findRoot(false)
 	if err != nil {
 		return nil, err
 	}
@@ -241,7 +237,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 
 	// Find the root node and check if it is a file or not
-	_, err = f.findRoot(ctx, false)
+	_, err = f.findRoot(false)
 	switch err {
 	case nil:
 		// root node found and is a directory
@@ -311,8 +307,8 @@ func (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err
 // lookupDir looks up the node for the directory of the name given
 //
 // if create is true it tries to create the root directory if not found
-func (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {
-	rootNode, err := f.findRoot(ctx, false)
+func (f *Fs) lookupDir(dir string) (*mega.Node, error) {
+	rootNode, err := f.findRoot(false)
 	if err != nil {
 		return nil, err
 	}
@@ -320,15 +316,15 @@ func (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {
 }
 
 // lookupParentDir finds the parent node for the remote passed in
-func (f *Fs) lookupParentDir(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
+func (f *Fs) lookupParentDir(remote string) (dirNode *mega.Node, leaf string, err error) {
 	parent, leaf := path.Split(remote)
-	dirNode, err = f.lookupDir(ctx, parent)
+	dirNode, err = f.lookupDir(parent)
 	return dirNode, leaf, err
 }
 
 // mkdir makes the directory and any parent directories for the
 // directory of the name given
-func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *mega.Node, err error) {
+func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error) {
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
@@ -362,7 +358,7 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *
 		// create directory called name in node
 		err = f.pacer.Call(func() (bool, error) {
 			node, err = f.srv.CreateDir(name, node)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "mkdir create node failed")
@@ -372,20 +368,20 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *
 }
 
 // mkdirParent creates the parent directory of remote
-func (f *Fs) mkdirParent(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
-	rootNode, err := f.findRoot(ctx, true)
+func (f *Fs) mkdirParent(remote string) (dirNode *mega.Node, leaf string, err error) {
+	rootNode, err := f.findRoot(true)
 	if err != nil {
 		return nil, "", err
 	}
 	parent, leaf := path.Split(remote)
-	dirNode, err = f.mkdir(ctx, rootNode, parent)
+	dirNode, err = f.mkdir(rootNode, parent)
 	return dirNode, leaf, err
 }
 
 // findRoot looks up the root directory node and returns it.
 //
 // if create is true it tries to create the root directory if not found
-func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
+func (f *Fs) findRoot(create bool) (*mega.Node, error) {
 	f.rootNodeMu.Lock()
 	defer f.rootNodeMu.Unlock()
 
@@ -407,7 +403,7 @@ func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
 	}
 
 	//..not found so create the root directory
-	f._rootNode, err = f.mkdir(ctx, absRoot, f.root)
+	f._rootNode, err = f.mkdir(absRoot, f.root)
 	return f._rootNode, err
 }
 
@@ -437,7 +433,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))
 		deleteErr := f.pacer.Call(func() (bool, error) {
 			err := f.srv.Delete(item, true)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if deleteErr != nil {
 			err = deleteErr
@@ -451,7 +447,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.Node) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -461,7 +457,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.No
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData(ctx) // reads info and meta, returning an error
+		err = o.readMetaData() // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -472,7 +468,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.No
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(ctx, remote, nil)
+	return f.newObjectWithInfo(remote, nil)
 }
 
 // list the objects into the function supplied
@@ -510,7 +506,7 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	dirNode, err := f.lookupDir(ctx, dir)
+	dirNode, err := f.lookupDir(dir)
 	if err != nil {
 		return nil, err
 	}
@@ -522,7 +518,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash())
 			entries = append(entries, d)
 		case mega.FILE:
-			o, err := f.newObjectWithInfo(ctx, remote, info)
+			o, err := f.newObjectWithInfo(remote, info)
 			if err != nil {
 				iErr = err
 				return true
@@ -546,8 +542,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Returns the dirNode, object, leaf and error
 //
 // Used to create new objects
-func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
-	dirNode, leaf, err = f.mkdirParent(ctx, remote)
+func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
+	dirNode, leaf, err = f.mkdirParent(remote)
 	if err != nil {
 		return nil, nil, leaf, err
 	}
@@ -569,7 +565,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
 		return existingObj, existingObj.Update(ctx, in, src, options...)
@@ -595,7 +591,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	size := src.Size()
 	modTime := src.ModTime(ctx)
 
-	o, _, _, err := f.createObject(ctx, remote, modTime, size)
+	o, _, _, err := f.createObject(remote, modTime, size)
 	if err != nil {
 		return nil, err
 	}
@@ -604,30 +600,30 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the directory if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	rootNode, err := f.findRoot(ctx, true)
+	rootNode, err := f.findRoot(true)
 	if err != nil {
 		return err
 	}
-	_, err = f.mkdir(ctx, rootNode, dir)
+	_, err = f.mkdir(rootNode, dir)
 	return errors.Wrap(err, "Mkdir failed")
 }
 
 // deleteNode removes a file or directory, observing useTrash
-func (f *Fs) deleteNode(ctx context.Context, node *mega.Node) (err error) {
+func (f *Fs) deleteNode(node *mega.Node) (err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		err = f.srv.Delete(node, f.opt.HardDelete)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	return err
 }
 
 // purgeCheck removes the directory dir, if check is set then it
 // refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
+func (f *Fs) purgeCheck(dir string, check bool) error {
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
-	rootNode, err := f.findRoot(ctx, false)
+	rootNode, err := f.findRoot(false)
 	if err != nil {
 		return err
 	}
@@ -648,7 +644,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 
 	waitEvent := f.srv.WaitEventsStart()
 
-	err = f.deleteNode(ctx, dirNode)
+	err = f.deleteNode(dirNode)
 	if err != nil {
 		return errors.Wrap(err, "delete directory node failed")
 	}
@@ -666,7 +662,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 //
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
+	return f.purgeCheck(dir, true)
 }
 
 // Precision return the precision of this Fs
@@ -680,13 +676,13 @@ func (f *Fs) Precision() time.Duration {
 // deleting all the files quicker than just running Remove() on the
 // result of List()
 func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+	return f.purgeCheck(dir, false)
 }
 
 // move a file or folder (srcFs, srcRemote, info) to (f, dstRemote)
 //
 // info will be updates
-func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
+func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
 	var (
 		dstFs                  = f
 		srcDirNode, dstDirNode *mega.Node
@@ -696,12 +692,12 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
 
 	if dstRemote != "" {
 		// lookup or create the destination parent directory
-		dstDirNode, dstLeaf, err = dstFs.mkdirParent(ctx, dstRemote)
+		dstDirNode, dstLeaf, err = dstFs.mkdirParent(dstRemote)
 	} else {
 		// find or create the parent of the root directory
 		absRoot := dstFs.srv.FS.GetRoot()
 		dstParent, dstLeaf = path.Split(dstFs.root)
-		dstDirNode, err = dstFs.mkdir(ctx, absRoot, dstParent)
+		dstDirNode, err = dstFs.mkdir(absRoot, dstParent)
 	}
 	if err != nil {
 		return errors.Wrap(err, "server-side move failed to make dst parent dir")
@@ -709,7 +705,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
 
 	if srcRemote != "" {
 		// lookup the existing parent directory
-		srcDirNode, srcLeaf, err = srcFs.lookupParentDir(ctx, srcRemote)
+		srcDirNode, srcLeaf, err = srcFs.lookupParentDir(srcRemote)
 	} else {
 		// lookup the existing root parent
 		absRoot := srcFs.srv.FS.GetRoot()
@@ -725,7 +721,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
 		//log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName())
 		err = f.pacer.Call(func() (bool, error) {
 			err = f.srv.Move(info, dstDirNode)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "server-side move failed")
@@ -739,7 +735,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
 		//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
 		err = f.pacer.Call(func() (bool, error) {
 			err = f.srv.Rename(info, f.opt.Enc.FromStandardName(dstLeaf))
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "server-side rename failed")
@@ -771,7 +767,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Do the move
-	err := f.move(ctx, remote, srcObj.fs, srcObj.remote, srcObj.info)
+	err := f.move(remote, srcObj.fs, srcObj.remote, srcObj.info)
 	if err != nil {
 		return nil, err
 	}
@@ -802,13 +798,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	// find the source
-	info, err := srcFs.lookupDir(ctx, srcRemote)
+	info, err := srcFs.lookupDir(srcRemote)
 	if err != nil {
 		return err
 	}
 
 	// check the destination doesn't exist
-	_, err = dstFs.lookupDir(ctx, dstRemote)
+	_, err = dstFs.lookupDir(dstRemote)
 	if err == nil {
 		return fs.ErrorDirExists
 	} else if err != fs.ErrorDirNotFound {
@@ -816,7 +812,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	// Do the move
-	err = f.move(ctx, dstRemote, srcFs, srcRemote, info)
+	err = f.move(dstRemote, srcFs, srcRemote, info)
 	if err != nil {
 		return err
 	}
@@ -842,7 +838,7 @@ func (f *Fs) Hashes() hash.Set {
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
-	root, err := f.findRoot(ctx, false)
+	root, err := f.findRoot(false)
 	if err != nil {
 		return "", errors.Wrap(err, "PublicLink failed to find root node")
 	}
@@ -890,7 +886,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 			fs.Infof(srcDir, "merging %q", f.opt.Enc.ToStandardName(info.GetName()))
 			err = f.pacer.Call(func() (bool, error) {
 				err = f.srv.Move(info, dstDirNode)
-				return shouldRetry(ctx, err)
+				return shouldRetry(err)
 			})
 			if err != nil {
 				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", f.opt.Enc.ToStandardName(info.GetName()), srcDir)
@@ -898,7 +894,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
-		err = f.deleteNode(ctx, srcDirNode)
+		err = f.deleteNode(srcDirNode)
 		if err != nil {
 			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
@@ -912,7 +908,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		q, err = f.srv.GetQuota()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get Mega Quota")
@@ -967,11 +963,11 @@ func (o *Object) setMetaData(info *mega.Node) (err error) {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // it also sets the info
-func (o *Object) readMetaData(ctx context.Context) (err error) {
+func (o *Object) readMetaData() (err error) {
 	if o.info != nil {
 		return nil
 	}
-	info, err := o.fs.readMetaDataForPath(ctx, o.remote)
+	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			err = fs.ErrorObjectNotFound
@@ -1002,7 +998,6 @@ func (o *Object) Storable() bool {
 
 // openObject represents a download in progress
 type openObject struct {
-	ctx    context.Context
 	mu     sync.Mutex
 	o      *Object
 	d      *mega.Download
@@ -1013,14 +1008,14 @@ type openObject struct {
 }
 
 // get the next chunk
-func (oo *openObject) getChunk(ctx context.Context) (err error) {
+func (oo *openObject) getChunk() (err error) {
 	if oo.id >= oo.d.Chunks() {
 		return io.EOF
 	}
 	var chunk []byte
 	err = oo.o.fs.pacer.Call(func() (bool, error) {
 		chunk, err = oo.d.DownloadChunk(oo.id)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -1050,7 +1045,7 @@ func (oo *openObject) Read(p []byte) (n int, err error) {
 		oo.skip -= int64(size)
 	}
 	if len(oo.chunk) == 0 {
-		err = oo.getChunk(oo.ctx)
+		err = oo.getChunk()
 		if err != nil {
 			return 0, err
 		}
@@ -1073,7 +1068,7 @@ func (oo *openObject) Close() (err error) {
 	}
 	err = oo.o.fs.pacer.Call(func() (bool, error) {
 		err = oo.d.Finish()
-		return shouldRetry(oo.ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to finish download")
@@ -1101,14 +1096,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var d *mega.Download
 	err = o.fs.pacer.Call(func() (bool, error) {
 		d, err = o.fs.srv.NewDownload(o.info)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open download file failed")
 	}
 
 	oo := &openObject{
-		ctx:  ctx,
 		o:    o,
 		d:    d,
 		skip: offset,
@@ -1131,7 +1125,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the parent directory
-	dirNode, leaf, err := o.fs.mkdirParent(ctx, remote)
+	dirNode, leaf, err := o.fs.mkdirParent(remote)
 	if err != nil {
 		return errors.Wrap(err, "update make parent dir failed")
 	}
@@ -1139,7 +1133,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var u *mega.Upload
 	err = o.fs.pacer.Call(func() (bool, error) {
 		u, err = o.fs.srv.NewUpload(dirNode, o.fs.opt.Enc.FromStandardName(leaf), size)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file failed to create session")
@@ -1160,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		err = o.fs.pacer.Call(func() (bool, error) {
 			err = u.UploadChunk(id, chunk)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "upload file failed to upload chunk")
@@ -1171,7 +1165,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var info *mega.Node
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, err = u.Finish()
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to finish upload")
@@ -1179,7 +1173,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// If the upload succeeded and the original object existed, then delete it
 	if o.info != nil {
-		err = o.fs.deleteNode(ctx, o.info)
+		err = o.fs.deleteNode(o.info)
 		if err != nil {
 			return errors.Wrap(err, "upload failed to remove old version")
 		}
@@ -1191,7 +1185,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	err := o.fs.deleteNode(ctx, o.info)
+	err := o.fs.deleteNode(o.info)
 	if err != nil {
 		return errors.Wrap(err, "Remove object failed")
 	}

backend/onedrive/onedrive.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -51,8 +52,8 @@ const (
 	driveTypePersonal           = "personal"
 	driveTypeBusiness           = "business"
 	driveTypeSharepoint         = "documentLibrary"
-	defaultChunkSize            = 10 * fs.Mebi
-	chunkSizeMultiple           = 320 * fs.Kibi
+	defaultChunkSize            = 10 * fs.MebiByte
+	chunkSizeMultiple           = 320 * fs.KibiByte
 
 	regionGlobal = "global"
 	regionUS     = "us"
@@ -93,12 +94,216 @@ var (
 
 // Register with Fs
 func init() {
-	QuickXorHashType = hash.RegisterHash("quickxor", "QuickXorHash", 40, quickxorhash.New)
+	QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "onedrive",
 		Description: "Microsoft OneDrive",
 		NewFs:       NewFs,
-		Config:      Config,
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+			region, _ := m.Get("region")
+			graphURL := graphAPIEndpoint[region] + "/v1.0"
+			oauthConfig.Endpoint = oauth2.Endpoint{
+				AuthURL:  authEndpoint[region] + authPath,
+				TokenURL: authEndpoint[region] + tokenPath,
+			}
+			ci := fs.GetConfig(ctx)
+			err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+				return
+			}
+
+			// Stop if we are running non-interactive config
+			if ci.AutoConfirm {
+				return
+			}
+
+			type driveResource struct {
+				DriveID   string `json:"id"`
+				DriveName string `json:"name"`
+				DriveType string `json:"driveType"`
+			}
+			type drivesResponse struct {
+				Drives []driveResource `json:"value"`
+			}
+
+			type siteResource struct {
+				SiteID   string `json:"id"`
+				SiteName string `json:"displayName"`
+				SiteURL  string `json:"webUrl"`
+			}
+			type siteResponse struct {
+				Sites []siteResource `json:"value"`
+			}
+
+			oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure OneDrive: %v", err)
+			}
+			srv := rest.NewClient(oAuthClient)
+
+			var opts rest.Opts
+			var finalDriveID string
+			var siteID string
+			var relativePath string
+			switch config.Choose("Your choice",
+				[]string{"onedrive", "sharepoint", "url", "search", "driveid", "siteid", "path"},
+				[]string{
+					"OneDrive Personal or Business",
+					"Root Sharepoint site",
+					"Sharepoint site name or URL (e.g. mysite or https://contoso.sharepoint.com/sites/mysite)",
+					"Search for a Sharepoint site",
+					"Type in driveID (advanced)",
+					"Type in SiteID (advanced)",
+					"Sharepoint server-relative path (advanced, e.g. /teams/hr)",
+				},
+				false) {
+
+			case "onedrive":
+				opts = rest.Opts{
+					Method:  "GET",
+					RootURL: graphURL,
+					Path:    "/me/drives",
+				}
+			case "sharepoint":
+				opts = rest.Opts{
+					Method:  "GET",
+					RootURL: graphURL,
+					Path:    "/sites/root/drives",
+				}
+			case "driveid":
+				fmt.Printf("Paste your Drive ID here> ")
+				finalDriveID = config.ReadLine()
+			case "siteid":
+				fmt.Printf("Paste your Site ID here> ")
+				siteID = config.ReadLine()
+			case "url":
+				fmt.Println("Example: \"https://contoso.sharepoint.com/sites/mysite\" or \"mysite\"")
+				fmt.Printf("Paste your Site URL here> ")
+				siteURL := config.ReadLine()
+				re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
+				match := re.FindStringSubmatch(siteURL)
+				if len(match) == 2 {
+					relativePath = "/sites/" + match[1]
+				} else {
+					relativePath = "/sites/" + siteURL
+				}
+			case "path":
+				fmt.Printf("Enter server-relative URL here> ")
+				relativePath = config.ReadLine()
+			case "search":
+				fmt.Printf("What to search for> ")
+				searchTerm := config.ReadLine()
+				opts = rest.Opts{
+					Method:  "GET",
+					RootURL: graphURL,
+					Path:    "/sites?search=" + searchTerm,
+				}
+
+				sites := siteResponse{}
+				_, err := srv.CallJSON(ctx, &opts, nil, &sites)
+				if err != nil {
+					log.Fatalf("Failed to query available sites: %v", err)
+				}
+
+				if len(sites.Sites) == 0 {
+					log.Fatalf("Search for '%s' returned no results", searchTerm)
+				} else {
+					fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
+					for index, site := range sites.Sites {
+						fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
+					}
+					siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
+				}
+			}
+
+			// if we use server-relative URL for finding the drive
+			if relativePath != "" {
+				opts = rest.Opts{
+					Method:  "GET",
+					RootURL: graphURL,
+					Path:    "/sites/root:" + relativePath,
+				}
+				site := siteResource{}
+				_, err := srv.CallJSON(ctx, &opts, nil, &site)
+				if err != nil {
+					log.Fatalf("Failed to query available site by relative path: %v", err)
+				}
+				siteID = site.SiteID
+			}
+
+			// if we have a siteID we need to ask for the drives
+			if siteID != "" {
+				opts = rest.Opts{
+					Method:  "GET",
+					RootURL: graphURL,
+					Path:    "/sites/" + siteID + "/drives",
+				}
+			}
+
+			// We don't have the final ID yet?
+			// query Microsoft Graph
+			if finalDriveID == "" {
+				drives := drivesResponse{}
+				_, err := srv.CallJSON(ctx, &opts, nil, &drives)
+				if err != nil {
+					log.Fatalf("Failed to query available drives: %v", err)
+				}
+
+				// Also call /me/drive as sometimes /me/drives doesn't return it #4068
+				if opts.Path == "/me/drives" {
+					opts.Path = "/me/drive"
+					meDrive := driveResource{}
+					_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
+					if err != nil {
+						log.Fatalf("Failed to query available drives: %v", err)
+					}
+					found := false
+					for _, drive := range drives.Drives {
+						if drive.DriveID == meDrive.DriveID {
+							found = true
+							break
+						}
+					}
+					// add the me drive if not found already
+					if !found {
+						fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
+						drives.Drives = append(drives.Drives, meDrive)
+					}
+				}
+
+				if len(drives.Drives) == 0 {
+					log.Fatalf("No drives found")
+				} else {
+					fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
+					for index, drive := range drives.Drives {
+						fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
+					}
+					finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
+				}
+			}
+
+			// Test the driveID and get drive type
+			opts = rest.Opts{
+				Method:  "GET",
+				RootURL: graphURL,
+				Path:    "/drives/" + finalDriveID + "/root"}
+			var rootItem api.Item
+			_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
+			if err != nil {
+				log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
+			}
+
+			fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
+			// This does not work, YET :)
+			if !config.ConfirmWithConfig(ctx, m, "config_drive_ok", true) {
+				log.Fatalf("Cancelled by user")
+			}
+
+			m.Set(configDriveID, finalDriveID)
+			m.Set(configDriveType, rootItem.ParentReference.DriveType)
+			config.SaveConfig()
+		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "region",
 			Help:    "Choose national cloud region for OneDrive.",
@@ -157,11 +362,6 @@ This will only work if you are copying between two OneDrive *Personal* drives AN
 the files to copy are already shared between them.  In other cases, rclone will
 fall back to normal copy (which will be slightly slower).`,
 			Advanced: true,
-		}, {
-			Name:     "list_chunk",
-			Help:     "Size of listing chunk.",
-			Default:  1000,
-			Advanced: true,
 		}, {
 			Name:    "no_versions",
 			Default: false,
@@ -261,266 +461,6 @@ At the time of writing this only works with OneDrive personal paid accounts.
 	})
 }
 
-type driveResource struct {
-	DriveID   string `json:"id"`
-	DriveName string `json:"name"`
-	DriveType string `json:"driveType"`
-}
-type drivesResponse struct {
-	Drives []driveResource `json:"value"`
-}
-
-type siteResource struct {
-	SiteID   string `json:"id"`
-	SiteName string `json:"displayName"`
-	SiteURL  string `json:"webUrl"`
-}
-type siteResponse struct {
-	Sites []siteResource `json:"value"`
-}
-
-// Get the region and graphURL from the config
-func getRegionURL(m configmap.Mapper) (region, graphURL string) {
-	region, _ = m.Get("region")
-	graphURL = graphAPIEndpoint[region] + "/v1.0"
-	return region, graphURL
-}
-
-// Config for chooseDrive
-type chooseDriveOpt struct {
-	opts         rest.Opts
-	finalDriveID string
-	siteID       string
-	relativePath string
-}
-
-// chooseDrive returns a query to choose which drive the user is interested in
-func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest.Client, opt chooseDriveOpt) (*fs.ConfigOut, error) {
-	_, graphURL := getRegionURL(m)
-
-	// if we use server-relative URL for finding the drive
-	if opt.relativePath != "" {
-		opt.opts = rest.Opts{
-			Method:  "GET",
-			RootURL: graphURL,
-			Path:    "/sites/root:" + opt.relativePath,
-		}
-		site := siteResource{}
-		_, err := srv.CallJSON(ctx, &opt.opts, nil, &site)
-		if err != nil {
-			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available site by relative path: %v", err))
-		}
-		opt.siteID = site.SiteID
-	}
-
-	// if we have a siteID we need to ask for the drives
-	if opt.siteID != "" {
-		opt.opts = rest.Opts{
-			Method:  "GET",
-			RootURL: graphURL,
-			Path:    "/sites/" + opt.siteID + "/drives",
-		}
-	}
-
-	drives := drivesResponse{}
-
-	// We don't have the final ID yet?
-	// query Microsoft Graph
-	if opt.finalDriveID == "" {
-		_, err := srv.CallJSON(ctx, &opt.opts, nil, &drives)
-		if err != nil {
-			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
-		}
-
-		// Also call /me/drive as sometimes /me/drives doesn't return it #4068
-		if opt.opts.Path == "/me/drives" {
-			opt.opts.Path = "/me/drive"
-			meDrive := driveResource{}
-			_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)
-			if err != nil {
-				return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
-			}
-			found := false
-			for _, drive := range drives.Drives {
-				if drive.DriveID == meDrive.DriveID {
-					found = true
-					break
-				}
-			}
-			// add the me drive if not found already
-			if !found {
-				fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
-				drives.Drives = append(drives.Drives, meDrive)
-			}
-		}
-	} else {
-		drives.Drives = append(drives.Drives, driveResource{
-			DriveID:   opt.finalDriveID,
-			DriveName: "Chosen Drive ID",
-			DriveType: "drive",
-		})
-	}
-	if len(drives.Drives) == 0 {
-		return fs.ConfigError("choose_type", "No drives found")
-	}
-	return fs.ConfigChoose("driveid_final", "config_driveid", "Select drive you want to use", len(drives.Drives), func(i int) (string, string) {
-		drive := drives.Drives[i]
-		return drive.DriveID, fmt.Sprintf("%s (%s)", drive.DriveName, drive.DriveType)
-	})
-}
-
-// Config the backend
-func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-	region, graphURL := getRegionURL(m)
-
-	if config.State == "" {
-		oauthConfig.Endpoint = oauth2.Endpoint{
-			AuthURL:  authEndpoint[region] + authPath,
-			TokenURL: authEndpoint[region] + tokenPath,
-		}
-		return oauthutil.ConfigOut("choose_type", &oauthutil.Options{
-			OAuth2Config: oauthConfig,
-		})
-	}
-
-	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure OneDrive")
-	}
-	srv := rest.NewClient(oAuthClient)
-
-	switch config.State {
-	case "choose_type":
-		return fs.ConfigChooseFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{
-			Value: "onedrive",
-			Help:  "OneDrive Personal or Business",
-		}, {
-			Value: "sharepoint",
-			Help:  "Root Sharepoint site",
-		}, {
-			Value: "url",
-			Help:  "Sharepoint site name or URL (e.g. mysite or https://contoso.sharepoint.com/sites/mysite)",
-		}, {
-			Value: "search",
-			Help:  "Search for a Sharepoint site",
-		}, {
-			Value: "driveid",
-			Help:  "Type in driveID (advanced)",
-		}, {
-			Value: "siteid",
-			Help:  "Type in SiteID (advanced)",
-		}, {
-			Value: "path",
-			Help:  "Sharepoint server-relative path (advanced, e.g. /teams/hr)",
-		}})
-	case "choose_type_done":
-		// Jump to next state according to config chosen
-		return fs.ConfigGoto(config.Result)
-	case "onedrive":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			opts: rest.Opts{
-				Method:  "GET",
-				RootURL: graphURL,
-				Path:    "/me/drives",
-			},
-		})
-	case "sharepoint":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			opts: rest.Opts{
-				Method:  "GET",
-				RootURL: graphURL,
-				Path:    "/sites/root/drives",
-			},
-		})
-	case "driveid":
-		return fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")
-	case "driveid_end":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			finalDriveID: config.Result,
-		})
-	case "siteid":
-		return fs.ConfigInput("siteid_end", "config_siteid", "Site ID")
-	case "siteid_end":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: config.Result,
-		})
-	case "url":
-		return fs.ConfigInput("url_end", "config_site_url", `Site URL
-
-Example: "https://contoso.sharepoint.com/sites/mysite" or "mysite"
-`)
-	case "url_end":
-		siteURL := config.Result
-		re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
-		match := re.FindStringSubmatch(siteURL)
-		if len(match) == 2 {
-			return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-				relativePath: "/sites/" + match[1],
-			})
-		}
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			relativePath: "/sites/" + siteURL,
-		})
-	case "path":
-		return fs.ConfigInput("path_end", "config_sharepoint_url", `Server-relative URL`)
-	case "path_end":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			relativePath: config.Result,
-		})
-	case "search":
-		return fs.ConfigInput("search_end", "config_search_term", `Search term`)
-	case "search_end":
-		searchTerm := config.Result
-		opts := rest.Opts{
-			Method:  "GET",
-			RootURL: graphURL,
-			Path:    "/sites?search=" + searchTerm,
-		}
-
-		sites := siteResponse{}
-		_, err := srv.CallJSON(ctx, &opts, nil, &sites)
-		if err != nil {
-			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available sites: %v", err))
-		}
-
-		if len(sites.Sites) == 0 {
-			return fs.ConfigError("choose_type", fmt.Sprintf("search for %q returned no results", searchTerm))
-		}
-		return fs.ConfigChoose("search_sites", "config_site", `Select the Site you want to use`, len(sites.Sites), func(i int) (string, string) {
-			site := sites.Sites[i]
-			return site.SiteID, fmt.Sprintf("%s (%s)", site.SiteName, site.SiteURL)
-		})
-	case "search_sites":
-		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: config.Result,
-		})
-	case "driveid_final":
-		finalDriveID := config.Result
-
-		// Test the driveID and get drive type
-		opts := rest.Opts{
-			Method:  "GET",
-			RootURL: graphURL,
-			Path:    "/drives/" + finalDriveID + "/root"}
-		var rootItem api.Item
-		_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
-		if err != nil {
-			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query root for drive %q: %v", finalDriveID, err))
-		}
-
-		m.Set(configDriveID, finalDriveID)
-		m.Set(configDriveType, rootItem.ParentReference.DriveType)
-
-		return fs.ConfigConfirm("driveid_final_end", true, "config_drive_ok", fmt.Sprintf("Drive OK?\n\nFound drive %q of type %q\nURL: %s\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL))
-	case "driveid_final_end":
-		if config.Result == "true" {
-			return nil, nil
-		}
-		return fs.ConfigGoto("choose_type")
-	}
-	return nil, fmt.Errorf("unknown state %q", config.State)
-}
-
 // Options defines the configuration for this backend
 type Options struct {
 	Region                  string               `config:"region"`
@@ -529,7 +469,6 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
-	ListChunk               int64                `config:"list_chunk"`
 	NoVersions              bool                 `config:"no_versions"`
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
@@ -611,10 +550,7 @@ var errAsyncJobAccessDenied = errors.New("async job failed - access denied")
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	retry := false
 	if resp != nil {
 		switch resp.StatusCode {
@@ -622,9 +558,6 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 			if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
 				retry = true
 				fs.Debugf(nil, "Should retry: %v", err)
-			} else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") {
-				retry = true
-				fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
 			}
 		case 429: // Too Many Requests.
 			// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
@@ -664,7 +597,7 @@ func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID s
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 
 	return info, resp, err
@@ -680,7 +613,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		opts.Path = strings.TrimSuffix(opts.Path, ":")
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		return info, resp, err
 	}
@@ -752,7 +685,7 @@ func errorHandler(resp *http.Response) error {
 }
 
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	const minChunkSize = fs.SizeSuffixBase
+	const minChunkSize = fs.Byte
 	if cs%chunkSizeMultiple != 0 {
 		return errors.Errorf("%s is not a multiple of %s", cs, chunkSizeMultiple)
 	}
@@ -936,7 +869,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -961,14 +894,14 @@ type listAllFn func(*api.Item) bool
 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	// Top parameter asks for bigger pages of data
 	// https://dev.onedrive.com/odata/optional-query-parameters.htm
-	opts := f.newOptsCall(dirID, "GET", fmt.Sprintf("/children?$top=%d", f.opt.ListChunk))
+	opts := f.newOptsCall(dirID, "GET", "/children?$top=1000")
 OUTER:
 	for {
 		var result api.ListChildrenResponse
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return found, errors.Wrap(err, "couldn't list files")
@@ -1105,7 +1038,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 
 	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
@@ -1155,7 +1088,7 @@ func (f *Fs) Precision() time.Duration {
 
 // waitForJob waits for the job with status in url to complete
 func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
-	deadline := time.Now().Add(f.ci.TimeoutOrInfinite())
+	deadline := time.Now().Add(f.ci.Timeout)
 	for time.Now().Before(deadline) {
 		var resp *http.Response
 		var err error
@@ -1193,7 +1126,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 
 		time.Sleep(1 * time.Second)
 	}
-	return errors.Errorf("async operation didn't complete after %v", f.ci.TimeoutOrInfinite())
+	return errors.Errorf("async operation didn't complete after %v", f.ci.Timeout)
 }
 
 // Copy src to this remote using server-side copy operations.
@@ -1261,7 +1194,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyReq, nil)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1354,7 +1287,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1421,7 +1354,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1447,7 +1380,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -1488,7 +1421,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		Password: f.opt.LinkPassword,
 	}
 
-	if expire < fs.DurationOff {
+	if expire < fs.Duration(time.Hour*24*365*100) {
 		expiry := time.Now().Add(time.Duration(expire))
 		share.Expiry = &expiry
 	}
@@ -1497,88 +1430,13 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var result api.CreateShareLinkResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &share, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
-		if resp != nil && resp.StatusCode == 400 && f.driveType != driveTypePersonal {
-			return "", errors.Errorf("%v (is making public links permitted by the org admin?)", err)
-		}
+		fmt.Println(err)
 		return "", err
 	}
-
-	shareURL := result.Link.WebURL
-
-	// Convert share link to direct download link if target is not a folder
-	// Not attempting to do the conversion for regional versions, just to be safe
-	if f.opt.Region != regionGlobal {
-		return shareURL, nil
-	}
-	if info.Folder != nil {
-		fs.Debugf(nil, "Can't convert share link for folder to direct link - returning the link as is")
-		return shareURL, nil
-	}
-
-	cnvFailMsg := "Don't know how to convert share link to direct link - returning the link as is"
-	directURL := ""
-	segments := strings.Split(shareURL, "/")
-	switch f.driveType {
-	case driveTypePersonal:
-		// Method: https://stackoverflow.com/questions/37951114/direct-download-link-to-onedrive-file
-		if len(segments) != 5 {
-			fs.Logf(f, cnvFailMsg)
-			return shareURL, nil
-		}
-		enc := base64.StdEncoding.EncodeToString([]byte(shareURL))
-		enc = strings.ReplaceAll(enc, "/", "_")
-		enc = strings.ReplaceAll(enc, "+", "-")
-		enc = strings.ReplaceAll(enc, "=", "")
-		directURL = fmt.Sprintf("https://api.onedrive.com/v1.0/shares/u!%s/root/content", enc)
-	case driveTypeBusiness:
-		// Method: https://docs.microsoft.com/en-us/sharepoint/dev/spfx/shorter-share-link-format
-		// Example:
-		//   https://{tenant}-my.sharepoint.com/:t:/g/personal/{user_email}/{Opaque_String}
-		//   --convert to->
-		//   https://{tenant}-my.sharepoint.com/personal/{user_email}/_layouts/15/download.aspx?share={Opaque_String}
-		if len(segments) != 8 {
-			fs.Logf(f, cnvFailMsg)
-			return shareURL, nil
-		}
-		directURL = fmt.Sprintf("https://%s/%s/%s/_layouts/15/download.aspx?share=%s",
-			segments[2], segments[5], segments[6], segments[7])
-	case driveTypeSharepoint:
-		// Method: Similar to driveTypeBusiness
-		// Example:
-		//   https://{tenant}.sharepoint.com/:t:/s/{site_name}/{Opaque_String}
-		//   --convert to->
-		//   https://{tenant}.sharepoint.com/sites/{site_name}/_layouts/15/download.aspx?share={Opaque_String}
-		//
-		//   https://{tenant}.sharepoint.com/:t:/t/{team_name}/{Opaque_String}
-		//   --convert to->
-		//   https://{tenant}.sharepoint.com/teams/{team_name}/_layouts/15/download.aspx?share={Opaque_String}
-		//
-		//   https://{tenant}.sharepoint.com/:t:/g/{Opaque_String}
-		//   --convert to->
-		//   https://{tenant}.sharepoint.com/_layouts/15/download.aspx?share={Opaque_String}
-		if len(segments) < 6 || len(segments) > 7 {
-			fs.Logf(f, cnvFailMsg)
-			return shareURL, nil
-		}
-		pathPrefix := ""
-		switch segments[4] {
-		case "s": // Site
-			pathPrefix = "/sites/" + segments[5]
-		case "t": // Team
-			pathPrefix = "/teams/" + segments[5]
-		case "g": // Root site
-		default:
-			fs.Logf(f, cnvFailMsg)
-			return shareURL, nil
-		}
-		directURL = fmt.Sprintf("https://%s%s/_layouts/15/download.aspx?share=%s",
-			segments[2], pathPrefix, segments[len(segments)-1])
-	}
-
-	return directURL, nil
+	return result.Link.WebURL, nil
 }
 
 // CleanUp deletes all the hidden files.
@@ -1617,7 +1475,7 @@ func (o *Object) deleteVersions(ctx context.Context) error {
 	var versions api.VersionsResponse
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1644,7 +1502,7 @@ func (o *Object) deleteVersion(ctx context.Context, ID string) error {
 	opts.NoResponse = true
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
@@ -1795,7 +1653,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	// Remove versions if required
 	if o.fs.opt.NoVersions {
@@ -1837,7 +1695,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1865,7 +1723,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 				err = errors.New(err.Error() + " (is it a OneNote file?)")
 			}
 		}
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return response, err
 }
@@ -1880,7 +1738,7 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return 0, err
@@ -1940,11 +1798,11 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			return true, errors.Wrapf(err, "retry this chunk skipping %d bytes", skip)
 		}
 		if err != nil {
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		}
 		body, err = rest.ReadBody(resp)
 		if err != nil {
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		}
 		if resp.StatusCode == 200 || resp.StatusCode == 201 {
 			// we are done :)
@@ -1967,7 +1825,7 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	return
 }
@@ -1991,7 +1849,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		fs.Debugf(o, "Cancelling multipart upload: %v", err)
 		cancelErr := o.cancelUploadSession(ctx, uploadURL)
 		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v (upload failed due to: %v)", cancelErr, err)
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
 	})()
 
@@ -2016,11 +1874,11 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 	return info, nil
 }
 
-// Update the content of a remote file within 4 MiB size in one single request
+// Update the content of a remote file within 4MB size in one single request
 // This function will set modtime after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
-		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
+		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -2038,7 +1896,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 				err = errors.New(err.Error() + " (is it a OneNote file?)")
 			}
 		}
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err

backend/opendrive/opendrive.go

@@ -88,7 +88,7 @@ func init() {
 
 Note that these chunks are buffered in memory so increasing them will
 increase memory use.`,
-			Default:  10 * fs.Mebi,
+			Default:  10 * fs.MebiByte,
 			Advanced: true,
 		}},
 	})
@@ -119,7 +119,6 @@ type Object struct {
 	fs      *Fs       // what this object is part of
 	remote  string    // The remote path
 	id      string    // ID of the file
-	parent  string    // ID of the parent directory
 	modTime time.Time // The modified time of the object if known
 	md5     string    // MD5 hash if known
 	size    int64     // Size of the object
@@ -207,7 +206,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			Path:   "/session/login.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create session")
@@ -234,7 +233,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(ctx, remote, nil, "")
+		_, err := tempF.newObjectWithInfo(ctx, remote, nil)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -294,7 +293,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 			Path:       "/folder/remove.json",
 		}
 		resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 }
 
@@ -389,7 +388,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -446,7 +445,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Path:   "/file/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -495,7 +494,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			Path:   "/folder/move_copy.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		fs.Debugf(src, "DirMove error %v", err)
@@ -518,7 +517,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, parent string) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (fs.Object, error) {
 	// fs.Debugf(nil, "newObjectWithInfo(%s, %v)", remote, file)
 
 	var o *Object
@@ -527,7 +526,6 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, p
 			fs:      f,
 			remote:  remote,
 			id:      file.FileID,
-			parent:  parent,
 			modTime: time.Unix(file.DateModified, 0),
 			size:    file.Size,
 			md5:     file.FileHash,
@@ -550,7 +548,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, p
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	// fs.Debugf(nil, "NewObject(\"%s\")", remote)
-	return f.newObjectWithInfo(ctx, remote, nil, "")
+	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
 // Creates from the parameters passed in a half finished Object which
@@ -583,7 +581,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -633,7 +631,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 				Path:    "/upload/create_file.json",
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
-			return o.fs.shouldRetry(ctx, resp, err)
+			return o.fs.shouldRetry(resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create file")
@@ -659,10 +657,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -688,7 +683,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			Path:   "/folder.json",
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -716,7 +711,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 			Path:   "/folder/list.json/" + f.session.SessionID + "/" + pathID,
 		}
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", false, errors.Wrap(err, "failed to get folder list")
@@ -759,7 +754,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	folderList := FolderList{}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get folder list")
@@ -773,7 +768,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		f.dirCache.Put(remote, folder.FolderID)
 		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
-		d.SetParentID(directoryID)
 		entries = append(entries, d)
 	}
 
@@ -781,7 +775,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		file.Name = f.opt.Enc.ToStandardName(file.Name)
 		// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
 		remote := path.Join(dir, file.Name)
-		o, err := f.newObjectWithInfo(ctx, remote, &file, directoryID)
+		o, err := f.newObjectWithInfo(ctx, remote, &file)
 		if err != nil {
 			return nil, err
 		}
@@ -848,7 +842,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 
 	o.modTime = modTime
@@ -868,7 +862,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open file)")
@@ -887,7 +881,7 @@ func (o *Object) Remove(ctx context.Context) error {
 			Path:       "/file.json/" + o.fs.session.SessionID + "/" + o.id,
 		}
 		resp, err := o.fs.srv.Call(ctx, &opts)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 }
 
@@ -916,7 +910,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:    "/upload/open_file_upload.json",
 		}
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -960,7 +954,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
-			return o.fs.shouldRetry(ctx, resp, err)
+			return o.fs.shouldRetry(resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to create file")
@@ -983,7 +977,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:   "/upload/close_file_upload.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create file")
@@ -1009,7 +1003,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Path:       "/file/access.json",
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1035,7 +1029,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 				o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
-		return o.fs.shouldRetry(ctx, resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to get folder list")
@@ -1059,11 +1053,6 @@ func (o *Object) ID() string {
 	return o.id
 }
 
-// ParentID returns the ID of the Object parent directory if known, or "" if not
-func (o *Object) ParentID() string {
-	return o.parent
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1074,5 +1063,4 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
-	_ fs.ParentIDer      = (*Object)(nil)
 )

backend/pcloud/pcloud.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"path"
@@ -71,7 +72,7 @@ func init() {
 		Name:        "pcloud",
 		Description: "Pcloud",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
 			optc := new(Options)
 			err := configstruct.Set(m, optc)
 			if err != nil {
@@ -93,11 +94,14 @@ func init() {
 				fs.Debugf(nil, "pcloud: got hostname %q", hostname)
 				return nil
 			}
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
+			opt := oauthutil.Options{
 				CheckAuth:    checkAuth,
 				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
-			})
+			}
+			err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     config.ConfigEncoding,
@@ -209,10 +213,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	doRetry := false
 
 	// Check if it is an api.Error
@@ -404,7 +405,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -459,7 +460,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -596,7 +597,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -661,7 +662,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -699,7 +700,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 
@@ -739,7 +740,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -786,7 +787,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -813,7 +814,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -837,7 +838,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -868,7 +869,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
 		err = q.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -926,7 +927,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -1045,7 +1046,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", err
@@ -1071,7 +1072,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1092,10 +1093,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	modTime := src.ModTime(ctx)
 	remote := o.Remote()
 
-	if size < 0 {
-		return errors.New("can't upload unknown sizes objects")
-	}
-
 	// Create the directory for the object if it doesn't exist
 	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
@@ -1137,7 +1134,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// opts.Body=0), so upload it as a multipart form POST with
 	// Content-Length set.
 	if size == 0 {
-		formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, opts.Parameters, "content", leaf)
+		formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
 		if err != nil {
 			return errors.Wrap(err, "failed to make multipart upload for 0 length file")
 		}
@@ -1154,18 +1151,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		// sometimes pcloud leaves a half complete file on
-		// error, so delete it if it exists, trying a few times
-		for i := 0; i < 5; i++ {
-			delObj, delErr := o.fs.NewObject(ctx, o.remote)
-			if delErr == nil && delObj != nil {
-				_ = delObj.Remove(ctx)
-				break
-			}
-			time.Sleep(time.Second)
+		// error, so delete it if it exists
+		delObj, delErr := o.fs.NewObject(ctx, o.remote)
+		if delErr == nil && delObj != nil {
+			_ = delObj.Remove(ctx)
 		}
 		return err
 	}
@@ -1188,7 +1181,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 }
 

backend/premiumizeme/premiumizeme.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
@@ -77,10 +78,11 @@ func init() {
 		Name:        "premiumizeme",
 		Description: "premiumize.me",
 		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
-			})
+		Config: func(ctx context.Context, name string, m configmap.Mapper) {
+			err := oauthutil.Config(ctx, "premiumizeme", name, m, oauthConfig, nil)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
 		},
 		Options: []fs.Option{{
 			Name: "api_key",
@@ -174,10 +176,7 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -371,7 +370,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		//fmt.Printf("...Error %v\n", err)
@@ -408,7 +407,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return found, errors.Wrap(err, "couldn't list files")
@@ -582,7 +581,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -661,7 +660,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
 		var result api.Response
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Move http")
@@ -770,7 +769,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "CreateDir http")
@@ -897,7 +896,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -935,7 +934,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
 		if err != nil {
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		}
 		// Just check the download URL resolves - sometimes
 		// the URLs returned by premiumize.me don't resolve so
@@ -994,7 +993,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var result api.Response
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "upload file http")
@@ -1036,7 +1035,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rename http")
@@ -1061,7 +1060,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
 	var result api.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "remove http")

backend/putio/error.go

@@ -1,7 +1,6 @@
 package putio
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 
@@ -30,10 +29,7 @@ func (e *statusCodeError) Temporary() bool {
 
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+func shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
 	}

backend/putio/fs.go

@@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
 		entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	return itoa(entry.ID), err
 }
@@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing file: %d", fileID)
 		children, _, err = f.client.Files.List(ctx, fileID)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
@@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "listing files inside List: %d", parentID)
 		children, _, err = f.client.Files.List(ctx, parentID)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return
@@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting file: %d", fileID)
 		entry, err = f.client.Files.Get(ctx, fileID)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -282,10 +282,11 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID string, modTime time.Time, options []fs.OpenOption) (location string, err error) {
 	// defer log.Trace(f, "name=%v, size=%v, parentID=%v, modTime=%v", name, size, parentID, modTime.String())("location=%v, err=%v", location, &err)
 	err = f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "POST", "https://upload.put.io/files/", nil)
+		req, err := http.NewRequest("POST", "https://upload.put.io/files/", nil)
 		if err != nil {
 			return false, err
 		}
+		req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 		req.Header.Set("tus-resumable", "1.0.0")
 		req.Header.Set("upload-length", strconv.FormatInt(size, 10))
 		b64name := base64.StdEncoding.EncodeToString([]byte(f.opt.Enc.FromStandardName(name)))
@@ -295,7 +296,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 		req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
 		fs.OpenOptionAddHTTPHeaders(req.Header, options)
 		resp, err := f.oAuthClient.Do(req)
-		retry, err := shouldRetry(ctx, err)
+		retry, err := shouldRetry(err)
 		if retry {
 			return true, err
 		}
@@ -320,7 +321,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 		err = f.pacer.Call(func() (bool, error) {
 			fs.Debugf(f, "Sending zero length chunk")
 			_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		return
 	}
@@ -344,13 +345,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				// Get file offset and seek to the position
 				offset, err := f.getServerOffset(ctx, location)
 				if err != nil {
-					return shouldRetry(ctx, err)
+					return shouldRetry(err)
 				}
 				sentBytes := offset - chunkStart
 				fs.Debugf(f, "sentBytes: %d", sentBytes)
 				_, err = chunk.Seek(sentBytes, io.SeekStart)
 				if err != nil {
-					return shouldRetry(ctx, err)
+					return shouldRetry(err)
 				}
 				transferOffset = offset
 				reqSize = chunkSize - sentBytes
@@ -367,7 +368,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 				offsetMismatch = true
 				return true, errors.New("connection broken")
 			}
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return
@@ -427,19 +428,21 @@ func (f *Fs) transferChunk(ctx context.Context, location string, start int64, ch
 }
 
 func (f *Fs) makeUploadHeadRequest(ctx context.Context, location string) (*http.Request, error) {
-	req, err := http.NewRequestWithContext(ctx, "HEAD", location, nil)
+	req, err := http.NewRequest("HEAD", location, nil)
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	req.Header.Set("tus-resumable", "1.0.0")
 	return req, nil
 }
 
 func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) {
-	req, err := http.NewRequestWithContext(ctx, "PATCH", location, in)
+	req, err := http.NewRequest("PATCH", location, in)
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	req.Header.Set("tus-resumable", "1.0.0")
 	req.Header.Set("upload-offset", strconv.FormatInt(offset, 10))
 	req.Header.Set("content-length", strconv.FormatInt(length, 10))
@@ -479,7 +482,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		err = f.pacer.Call(func() (bool, error) {
 			// fs.Debugf(f, "listing files: %d", dirID)
 			children, _, err = f.client.Files.List(ctx, dirID)
-			return shouldRetry(ctx, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "Rmdir")
@@ -493,7 +496,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "deleting file: %d", dirID)
 		err = f.client.Files.Delete(ctx, dirID)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	f.dirCache.FlushDir(dir)
 	return err
@@ -552,7 +555,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -591,7 +594,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -631,7 +634,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	srcFs.dirCache.FlushDir(srcRemote)
 	return err
@@ -644,7 +647,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "getting account info")
 		ai, err = f.client.Account.Info(ctx)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about failed")
@@ -678,6 +681,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		}
 		// fs.Debugf(f, "emptying trash")
 		_, err = f.client.Do(req, nil)
-		return shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 }