protondrive: add SkipSignatureVerifications to option

This adds an additional parameter to the creation of each flag. This
specifies one or more flag groups. This **must** be set for global
flags and **must not** be set for local flags.

This causes flags.md to be built with sections to aid comprehension
and it causes the documentation pages for each command (and the
`--help`) to be built showing the flags groups as specified in the
`groups` annotation on the command.

See: https://forum.rclone.org/t/make-docs-for-mortals-not-only-rclone-gurus/39476/

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-github: [ncw]
-patreon: njcw
-liberapay: ncw
-custom: ["https://rclone.org/donate/"]

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -8,29 +8,31 @@ name: build
 on:
   push:
     branches:
-      - '*'
+      - '**'
     tags:
-      - '*'
+      - '**'
   pull_request:
   workflow_dispatch:
     inputs:
       manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
         required: true
         default: true
 
 jobs:
   build:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.16', 'go1.17']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.21.0-rc.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -39,9 +41,16 @@ jobs:
             librclonetest: true
             deploy: true
 
+          - job_name: linux_386
+            os: ubuntu-latest
+            go: '1.21.0-rc.3'
+            goarch: 386
+            gotags: cmount
+            quicktest: true
+
           - job_name: mac_amd64
             os: macos-11
-            go: '1.18.x'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -50,48 +59,37 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.18.x'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
-          - job_name: windows_amd64
+          - job_name: windows
             os: windows-latest
-            go: '1.18.x'
+            go: '1.21.0-rc.3'
             gotags: cmount
-            build_flags: '-include "^windows/amd64" -cgo'
-            build_args: '-buildmode exe'
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: windows_386
-            os: windows-latest
-            go: '1.18.x'
-            gotags: cmount
-            goarch: '386'
-            cgo: '1'
-            build_flags: '-include "^windows/386" -cgo'
+            cgo: '0'
+            build_flags: '-include "^windows/"'
             build_args: '-buildmode exe'
             quicktest: true
             deploy: true
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.18.x'
-            build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
+            go: '1.21.0-rc.3'
+            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.16
+          - job_name: go1.19
             os: ubuntu-latest
-            go: '1.16.x'
+            go: '1.19'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.17
+          - job_name: go1.20
             os: ubuntu-latest
-            go: '1.17.x'
+            go: '1.20'
             quicktest: true
             racequicktest: true
 
@@ -101,14 +99,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 
@@ -127,12 +124,17 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse libfuse-dev rpm pkg-config
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
         shell: bash
         run: |
+          # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
+          # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
+          unset HOMEBREW_NO_INSTALL_FROM_API
+          brew untap --force homebrew/core
+          brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
         if: matrix.os == 'macos-11'
@@ -166,7 +168,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -178,6 +180,11 @@ jobs:
         run: |
           make
 
+      - name: Rclone version
+        shell: bash
+        run: |
+          rclone version
+
       - name: Run tests
         shell: bash
         run: |
@@ -215,48 +222,57 @@ jobs:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
         # Deploy binaries if enabled in config && not a PR && not a fork
-        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "lint"
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Code quality test
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
+      # Run govulncheck on the latest go version, the one we build binaries with
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.0-rc.3'
+          check-latest: true
+
+      - name: Install govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+
+      - name: Scan for vulnerabilities
+        run: govulncheck ./...
+
   android:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "android-all"
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.18.x
-
-      # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
-      - name: Force NDK version
-        run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;23.1.7779620" | grep -v = || true
+          go-version: '1.21.0-rc.3'
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -277,27 +293,29 @@ jobs:
           go install golang.org/x/mobile/cmd/gobind@latest
           go install golang.org/x/mobile/cmd/gomobile@latest
           env PATH=$PATH:~/go/bin gomobile init
+          echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV
 
       - name: arm-v7a gomobile build
-        run: env PATH=$PATH:~/go/bin gomobile bind -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
+        run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
 
       - name: arm-v7a Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=arm' >> $GITHUB_ENV
           echo 'GOARM=7' >> $GITHUB_ENV
           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+
       - name: arm-v7a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
 
       - name: arm64-v8a Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=arm64' >> $GITHUB_ENV
@@ -305,12 +323,12 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: arm64-v8a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
 
       - name: x86 Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=386' >> $GITHUB_ENV
@@ -318,12 +336,12 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: x86 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
 
       - name: x64 Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=amd64' >> $GITHUB_ENV
@@ -331,7 +349,7 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: x64 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 .
 
       - name: Upload artifacts
         run: |
@@ -339,4 +357,4 @@ jobs:
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork
-        if: github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,61 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v2
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v4
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v2
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Build and publish image
+              uses: docker/build-push-action@v4
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha
+                cache-to: type=gha,mode=max
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252

.github/workflows/build_publish_docker_image.yml

@@ -1,26 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v2
-              with:
-                fetch-depth: 0
-            - name: Build and publish image
-              uses: ilteoood/docker_buildx@1.1.0
-              with:
-                tag: beta
-                imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
-                publish: true
-                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
-                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -28,7 +28,7 @@ jobs:
               with:
                 tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
                 imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
                 publish: true
                 dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
                 dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin
@@ -50,7 +50,7 @@ jobs:
                 PLUGIN_USER=rclone
                 docker login --username ${{ secrets.DOCKER_HUB_USER }} \
                              --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
                     export PLUGIN_USER PLUGIN_ARCH
                     make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
                     make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}

.github/workflows/winget.yml

@@ -0,0 +1,14 @@
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: windows-latest # Action can only run on Windows
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.golangci.yml

@@ -2,15 +2,17 @@
 
 linters:
   enable:
-    - deadcode
     - errcheck
     - goimports
     - revive
     - ineffassign
-    - structcheck
-    - varcheck
     - govet
     - unconvert
+    - staticcheck
+    - gosimple
+    - stylecheck
+    - unused
+    - misspell
     #- prealloc
     #- maligned
   disable-all: true
@@ -20,11 +22,35 @@ issues:
   exclude-use-default: false
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
+  max-issues-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
+  exclude-rules:
+
+    - linters:
+      - staticcheck
+      text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
+
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 10m
+
+linters-settings:
+  revive:
+    rules:
+      - name: unreachable-code
+        disabled: true
+      - name: unused-parameter
+        disabled: true
+      - name: empty-block
+        disabled: true
+      - name: redefines-builtin-id
+        disabled: true
+      - name: superfluous-else
+        disabled: true
+  stylecheck:
+    # Only enable the checks performed by the staticcheck stand-alone tool,
+    # as documented here: https://staticcheck.io/docs/configuration/options/#checks
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]

CONTRIBUTING.md

@@ -77,7 +77,7 @@ Make sure you
   * Add [documentation](#writing-documentation) for a new feature.
   * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
 
-When you are done with that push your changes to Github:
+When you are done with that push your changes to GitHub:
 
     git push -u origin my-new-feature
 
@@ -88,7 +88,7 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I
 
 You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
 
-## Using Git and Github ##
+## Using Git and GitHub ##
 
 ### Committing your changes ###
 
@@ -419,7 +419,7 @@ remote or an fs.
 
 Research
 
-  * Look at the interfaces defined in `fs/fs.go`
+  * Look at the interfaces defined in `fs/types.go`
   * Study one or more of the existing remotes
 
 Getting going

Dockerfile

@@ -11,7 +11,7 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse tzdata && \
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

MAINTAINERS.md

@@ -16,6 +16,9 @@ Current active maintainers of rclone are:
 | Max Sum          | @Max-Sum          | union backend                |
 | Fred             | @creativeprojects | seafile backend              |
 | Caleb Case       | @calebcase        | storj backend                |
+| wiserain         | @wiserain         | pikpak backend               |
+| albertony        | @albertony        |                              |
+| Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
 
 **This is a work in progress Draft**
 

Makefile

@@ -81,6 +81,9 @@ quicktest:
 racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
+compiletest:
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
+
 # Do source code quality checks
 check:	rclone
 	@echo "-- START CODE QUALITY REPORT -------------------------------"
@@ -93,11 +96,11 @@ build_dep:
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64\.tar\.gz'
+	go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
 
 # Get the release dependencies we only install on Windows
 release_dep_windows:
-	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
+	GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
 
 # Update dependencies
 showupdates:
@@ -245,18 +248,18 @@ retag:
 startdev:
 	@echo "Version is $(VERSION)"
 	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
 	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
 	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
 startstable:
 	@echo "Version is $(VERSION)"
 	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
 	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
 	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -25,28 +25,34 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
+  * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
-  * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
+  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
   * Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
+  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
+  * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
-  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
+  * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
+  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
@@ -56,23 +62,30 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
+  * Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
+  * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
+  * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
+  * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
+  * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * Storj [:page_facing_up:](https://rclone.org/storj/)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
+  * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -82,6 +95,19 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
 
 Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
 
+### Virtual storage providers
+
+These backends adapt or modify other storage providers
+
+  * Alias: rename existing remotes [:page_facing_up:](https://rclone.org/alias/)
+  * Cache: cache remotes (DEPRECATED) [:page_facing_up:](https://rclone.org/cache/)
+  * Chunker: split large files [:page_facing_up:](https://rclone.org/chunker/)
+  * Combine: combine multiple remotes into a directory tree [:page_facing_up:](https://rclone.org/combine/)
+  * Compress: compress files [:page_facing_up:](https://rclone.org/compress/)
+  * Crypt: encrypt files [:page_facing_up:](https://rclone.org/crypt/)
+  * Hasher: hash files [:page_facing_up:](https://rclone.org/hasher/)
+  * Union: join multiple remotes to work together [:page_facing_up:](https://rclone.org/union/)
+
 ## Features
 
   * MD5/SHA-1 hashes checked at all times for file integrity
@@ -96,7 +122,7 @@ Please see [the full list of all storage providers and their features](https://r
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
   * Multi-threaded downloads to local disk
-  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
+  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDAV/FTP/SFTP/DLNA
 
 ## Installation & documentation
 
@@ -117,5 +143,5 @@ Please see the [rclone website](https://rclone.org/) for:
 License
 -------
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 [COPYING file](/COPYING) included in this package).

RELEASE.md

@@ -10,7 +10,7 @@ This file describes how to make the various kinds of releases
 ## Making a release
 
   * git checkout master # see below for stable branch
-  * git pull
+  * git pull # IMPORTANT
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
@@ -21,6 +21,7 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
+  * git push origin # without --follow-tags so it doesn't push the tag if it fails
   * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
@@ -53,6 +54,14 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
+## Tidy beta
+
+At some point after the release run
+
+    bin/tidy-beta v1.55
+
+where the version number is that of a couple ago to remove old beta binaries.
+
 ## Making a point release
 
 If rclone needs a point release due to some horrendous bug:
@@ -66,8 +75,7 @@ Set vars
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * git branch ${BASE_TAG} ${BASE_TAG}-stable
-  * git co ${BASE_TAG}-stable
+  * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
   * make startstable
 
 Now

VERSION

@@ -1 +1 @@
-v1.59.0
+v1.64.0

backend/alias/alias.go

@@ -1,3 +1,4 @@
+// Package alias implements a virtual provider to rename existing remotes.
 package alias
 
 import (

backend/all/all.go

@@ -1,3 +1,4 @@
+// Package all imports all the backends
 package all
 
 import (
@@ -9,6 +10,7 @@ import (
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
+	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/drive"
@@ -20,8 +22,8 @@ import (
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
 	_ "github.com/rclone/rclone/backend/hdfs"
+	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/hubic"
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
@@ -32,8 +34,11 @@ import (
 	_ "github.com/rclone/rclone/backend/netstorage"
 	_ "github.com/rclone/rclone/backend/onedrive"
 	_ "github.com/rclone/rclone/backend/opendrive"
+	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
+	_ "github.com/rclone/rclone/backend/pikpak"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
+	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"
@@ -41,6 +46,7 @@ import (
 	_ "github.com/rclone/rclone/backend/sftp"
 	_ "github.com/rclone/rclone/backend/sharefile"
 	_ "github.com/rclone/rclone/backend/sia"
+	_ "github.com/rclone/rclone/backend/smb"
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"

backend/amazonclouddrive/amazonclouddrive.go

@@ -435,7 +435,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, title string, directorie
 		query += " AND kind:" + folderKind
 	} else if filesOnly {
 		query += " AND kind:" + fileKind
-	} else {
+		//} else {
 		// FIXME none of these work
 		//query += " AND kind:(" + fileKind + " OR " + folderKind + ")"
 		//query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")"
@@ -556,9 +556,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // This is a workaround for Amazon sometimes returning
 //
-//  * 408 REQUEST_TIMEOUT
-//  * 504 GATEWAY_TIMEOUT
-//  * 500 Internal server error
+//   - 408 REQUEST_TIMEOUT
+//   - 504 GATEWAY_TIMEOUT
+//   - 500 Internal server error
 //
 // At the end of large uploads.  The speculation is that the timeout
 // is waiting for the sha1 hashing to complete and the file may well
@@ -626,7 +626,7 @@ func (f *Fs) checkUpload(ctx context.Context, resp *http.Response, in io.Reader,
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -685,9 +685,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1002,7 +1002,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {

backend/azureblob/azureblob_test.go

@@ -6,10 +6,10 @@
 package azureblob
 
 import (
-	"context"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
 )
@@ -17,10 +17,31 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:    "TestAzureBlob:",
-		NilObject:     (*Object)(nil),
-		TiersToTest:   []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{},
+		RemoteName:  "TestAzureBlob:",
+		NilObject:   (*Object)(nil),
+		TiersToTest: []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: defaultChunkSize,
+		},
+	})
+}
+
+// TestIntegration2 runs integration tests against the remote
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	name := "TestAzureBlob:"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:  name,
+		NilObject:   (*Object)(nil),
+		TiersToTest: []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: defaultChunkSize,
+		},
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "directory_markers", Value: "true"},
+		},
 	})
 }
 
@@ -32,32 +53,24 @@ var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 )
 
-// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
-func TestServicePrincipalFileSuccess(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "password": "my secret",
-    "tenant": "my active directory tenant ID"
-}
-`
-	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	if assert.NoError(t, err) {
-		assert.NotNil(t, tokenRefresher)
+func TestValidateAccessTier(t *testing.T) {
+	tests := map[string]struct {
+		accessTier string
+		want       bool
+	}{
+		"hot":     {"hot", true},
+		"HOT":     {"HOT", true},
+		"Hot":     {"Hot", true},
+		"cool":    {"cool", true},
+		"archive": {"archive", true},
+		"empty":   {"", false},
+		"unknown": {"unknown", false},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			got := validateAccessTier(test.accessTier)
+			assert.Equal(t, test.want, got)
+		})
 	}
 }
-
-// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
-func TestServicePrincipalFileFailure(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "tenant": "my active directory tenant ID"
-}
-`
-	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	assert.Error(t, err)
-	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
-}

backend/azureblob/imds.go

@@ -1,137 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-)
-
-const (
-	azureResource      = "https://storage.azure.com"
-	imdsAPIVersion     = "2018-02-01"
-	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
-)
-
-// This custom type is used to add the port the test server has bound to
-// to the request context.
-type testPortKey string
-
-type msiIdentifierType int
-
-const (
-	msiClientID msiIdentifierType = iota
-	msiObjectID
-	msiResourceID
-)
-
-type userMSI struct {
-	Type  msiIdentifierType
-	Value string
-}
-
-type httpError struct {
-	Response *http.Response
-}
-
-func (e httpError) Error() string {
-	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
-}
-
-// GetMSIToken attempts to obtain an MSI token from the Azure Instance
-// Metadata Service.
-func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
-	// Attempt to get an MSI token; silently continue if unsuccessful.
-	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
-	result := adal.Token{}
-	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
-	if err != nil {
-		fs.Debugf(nil, "Failed to create request: %v", err)
-		return result, err
-	}
-	params := req.URL.Query()
-	params.Set("resource", azureResource)
-	params.Set("api-version", imdsAPIVersion)
-
-	// Specify user-assigned identity if requested.
-	if identity != nil {
-		switch identity.Type {
-		case msiClientID:
-			params.Set("client_id", identity.Value)
-		case msiObjectID:
-			params.Set("object_id", identity.Value)
-		case msiResourceID:
-			params.Set("mi_res_id", identity.Value)
-		default:
-			// If this happens, the calling function and this one don't agree on
-			// what valid ID types exist.
-			return result, fmt.Errorf("unknown MSI identity type specified")
-		}
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// The Metadata header is required by all calls to IMDS.
-	req.Header.Set("Metadata", "true")
-
-	// If this function is run in a test, query the test server instead of IMDS.
-	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
-	if isTest {
-		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
-		req.Host = req.URL.Host
-	}
-
-	// Send request
-	httpClient := fshttp.NewClient(ctx)
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
-	}
-	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
-		if err != nil {
-			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
-		}
-		err = resp.Body.Close()
-		if err != nil {
-			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
-		}
-	}()
-	// Check if the status code indicates success
-	// The request returns 200 currently, add 201 and 202 as well for possible extension.
-	switch resp.StatusCode {
-	case 200, 201, 202:
-		break
-	default:
-		body, _ := ioutil.ReadAll(resp.Body)
-		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
-		return result, httpError{Response: resp}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("Couldn't read IMDS response: %w", err)
-	}
-	// Remove BOM, if any. azcopy does this so I'm following along.
-	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
-
-	// This would be a good place to persist the token if a large number of rclone
-	// invocations are being made in a short amount of time. If the token is
-	// persisted, the azureblob code will need to check for expiry before every
-	// storage API call.
-	err = json.Unmarshal(b, &result)
-	if err != nil {
-		return result, fmt.Errorf("Couldn't unmarshal IMDS response: %w", err)
-	}
-
-	return result, nil
-}

backend/azureblob/imds_test.go

@@ -1,118 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		err := r.ParseForm()
-		require.NoError(t, err)
-		parameters := r.URL.Query()
-		(*actual)["path"] = r.URL.Path
-		(*actual)["Metadata"] = r.Header.Get("Metadata")
-		(*actual)["method"] = r.Method
-		for paramName := range parameters {
-			(*actual)[paramName] = parameters.Get(paramName)
-		}
-		// Make response.
-		response := adal.Token{}
-		responseBytes, err := json.Marshal(response)
-		require.NoError(t, err)
-		_, err = w.Write(responseBytes)
-		require.NoError(t, err)
-	}
-}
-
-func TestManagedIdentity(t *testing.T) {
-	// test user-assigned identity specifiers to use
-	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
-	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
-	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
-	tests := []struct {
-		identity              *userMSI
-		identityParameterName string
-		expectedAbsent        []string
-	}{
-		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
-		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
-		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
-		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
-	}
-	alwaysExpected := map[string]string{
-		"path":        "/metadata/identity/oauth2/token",
-		"resource":    "https://storage.azure.com",
-		"Metadata":    "true",
-		"api-version": "2018-02-01",
-		"method":      "GET",
-	}
-	for _, test := range tests {
-		actual := make(map[string]string, 10)
-		testServer := httptest.NewServer(handler(t, &actual))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, test.identity)
-		require.NoError(t, err)
-
-		// Validate expected query parameters present
-		expected := make(map[string]string)
-		for k, v := range alwaysExpected {
-			expected[k] = v
-		}
-		if test.identity != nil {
-			expected[test.identityParameterName] = test.identity.Value
-		}
-
-		for key := range expected {
-			value, exists := actual[key]
-			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
-				test.identityParameterName, key) {
-				assert.Equalf(t, expected[key], value,
-					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
-			}
-		}
-
-		// Validate unexpected query parameters absent
-		for _, key := range test.expectedAbsent {
-			_, exists := actual[key]
-			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
-		}
-	}
-}
-
-func errorHandler(resultCode int) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Test error generated", resultCode)
-	}
-}
-
-func TestIMDSErrors(t *testing.T) {
-	errorCodes := []int{404, 429, 500}
-	for _, code := range errorCodes {
-		testServer := httptest.NewServer(errorHandler(code))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, nil)
-		require.Error(t, err)
-		httpErr, ok := err.(httpError)
-		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
-		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
-	}
-}

backend/b2/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Backblaze B2 API.
 package api
 
 import (
@@ -238,7 +239,7 @@ type GetFileInfoRequest struct {
 // If the original source of the file being uploaded has a last
 // modified time concept, Backblaze recommends using
 // src_last_modified_millis as the name, and a string holding the base
-// 10 number number of milliseconds since midnight, January 1, 1970
+// 10 number of milliseconds since midnight, January 1, 1970
 // UTC. This fits in a 64 bit integer such as the type "long" in the
 // programming language Java. It is intended to be compatible with
 // Java's time long. For example, it can be passed directly into the

backend/b2/b2.go

@@ -1,4 +1,4 @@
-// Package b2 provides an interface to the Backblaze B2 object storage system
+// Package b2 provides an interface to the Backblaze B2 object storage system.
 package b2
 
 // FIXME should we remove sha1 checks from here as rclone now supports
@@ -64,7 +64,8 @@ const (
 
 // Globals
 var (
-	errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersions  = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
 )
 
 // Register with Fs
@@ -74,13 +75,15 @@ func init() {
 		Description: "Backblaze B2",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "account",
-			Help:     "Account ID or Application Key ID.",
-			Required: true,
+			Name:      "account",
+			Help:      "Account ID or Application Key ID.",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:     "key",
-			Help:     "Application Key.",
-			Required: true,
+			Name:      "key",
+			Help:      "Application Key.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -106,6 +109,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 			Help:     "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "version_at",
+			Help:     "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "hard_delete",
 			Help:    "Permanently delete files on remote removal, otherwise hide files.",
@@ -211,6 +219,7 @@ type Options struct {
 	Endpoint                      string               `config:"endpoint"`
 	TestMode                      string               `config:"test_mode"`
 	Versions                      bool                 `config:"versions"`
+	VersionAt                     fs.Time              `config:"version_at"`
 	HardDelete                    bool                 `config:"hard_delete"`
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
@@ -273,7 +282,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootBucket == "" {
-		return fmt.Sprintf("B2 root")
+		return "B2 root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("B2 bucket %s", f.rootBucket)
@@ -649,15 +658,15 @@ var errEndList = errors.New("end list")
 //
 // (bucket, directory) is the starting directory
 //
-// If prefix is set then it is removed from all file names
+// If prefix is set then it is removed from all file names.
 //
 // If addBucket is set then it adds the bucket to the start of the
-// remotes generated
+// remotes generated.
 //
-// If recurse is set the function will recursively list
+// If recurse is set the function will recursively list.
 //
 // If limit is > 0 then it limits to that many files (must be less
-// than 1000)
+// than 1000).
 //
 // If hidden is set then it will list the hidden (deleted) files too.
 //
@@ -696,9 +705,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		Method: "POST",
 		Path:   "/b2_list_file_names",
 	}
-	if hidden {
+	if hidden || f.opt.VersionAt.IsSet() {
 		opts.Path = "/b2_list_file_versions"
 	}
+
+	lastFileName := ""
+
 	for {
 		var response api.ListFileNamesResponse
 		err := f.pacer.Call(func() (bool, error) {
@@ -728,7 +740,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
+
+			if f.opt.VersionAt.IsSet() {
+				if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
+					// Ignore versions that were created after the specified time
+					continue
+				}
+
+				if file.Name == lastFileName {
+					// Ignore versions before the already returned version
+					continue
+				}
+			}
+
 			// Send object
+			lastFileName = file.Name
 			err = fn(remote, file, isDirectory)
 			if err != nil {
 				if err == errEndList {
@@ -1001,7 +1027,7 @@ func (f *Fs) clearBucketID(bucket string) {
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1181,10 +1207,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 		}
 	}
 	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		if time.Since(time.Time(timestamp)).Hours() > 24 {
-			return true
-		}
-		return false
+		return time.Since(time.Time(timestamp)).Hours() > 24
 	}
 
 	// Delete Config.Transfers in parallel
@@ -1200,7 +1223,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 					fs.Errorf(object.Name, "Can't create object %v", err)
 					continue
 				}
-				tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
+				tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
 				err = f.deleteByID(ctx, object.ID, object.Name)
 				checkErr(err)
 				tr.Done(ctx, err)
@@ -1214,7 +1237,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			if err != nil {
 				fs.Errorf(object, "Can't create object %+v", err)
 			}
-			tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
+			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 			if oldOnly && last != remote {
 				// Check current version of the file
 				if object.Action == "hide" {
@@ -1313,9 +1336,9 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1457,26 +1480,23 @@ func (o *Object) Size() int64 {
 
 // Clean the SHA1
 //
-// Make sure it is lower case
+// Make sure it is lower case.
 //
 // Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
 // Some tools (e.g. Cyberduck) use this
-func cleanSHA1(sha1 string) (out string) {
-	out = strings.ToLower(sha1)
+func cleanSHA1(sha1 string) string {
 	const unverified = "unverified:"
-	if strings.HasPrefix(out, unverified) {
-		out = out[len(unverified):]
-	}
-	return out
+	return strings.TrimPrefix(strings.ToLower(sha1), unverified)
 }
 
 // decodeMetaDataRaw sets the metadata from the data passed in
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) {
 	o.id = ID
 	o.sha1 = SHA1
@@ -1495,10 +1515,11 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 // decodeMetaData sets the metadata in the object from an api.File
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaData(info *api.File) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1506,10 +1527,11 @@ func (o *Object) decodeMetaData(info *api.File) (err error) {
 // decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1567,10 +1589,11 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.id != "" {
 		return nil
@@ -1828,6 +1851,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	size := src.Size()
 
 	bucket, bucketPath := o.split()
@@ -1983,6 +2009,9 @@ func (o *Object) Remove(ctx context.Context) error {
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	if o.fs.opt.HardDelete {
 		return o.fs.deleteByID(ctx, o.id, bucketPath)
 	}

backend/b2/upload.go

@@ -14,12 +14,15 @@ import (
 	"io"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/sync/errgroup"
 )
@@ -88,21 +91,19 @@ type largeUpload struct {
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
-	remote := o.remote
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
 	parts := int64(0)
 	sha1SliceSize := int64(maxParts)
+	chunkSize := defaultChunkSize
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
+		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
 		parts = size / int64(chunkSize)
 		if size%int64(chunkSize) != 0 {
 			parts++
 		}
-		if parts > maxParts {
-			return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
-		}
 		sha1SliceSize = parts
 	}
 
@@ -429,18 +430,47 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
 	var (
-		g, gCtx   = errgroup.WithContext(ctx)
-		remaining = up.size
+		g, gCtx    = errgroup.WithContext(ctx)
+		remaining  = up.size
+		uploadPool *pool.Pool
+		ci         = fs.GetConfig(ctx)
 	)
+	// If using large chunk size then make a temporary pool
+	if up.chunkSize <= int64(up.f.opt.ChunkSize) {
+		uploadPool = up.f.pool
+	} else {
+		uploadPool = pool.New(
+			time.Duration(up.f.opt.MemoryPoolFlushTime),
+			int(up.chunkSize),
+			ci.Transfers,
+			up.f.opt.MemoryPoolUseMmap,
+		)
+		defer uploadPool.Flush()
+	}
+	// Get an upload token and a buffer
+	getBuf := func() (buf []byte) {
+		up.f.getBuf(true)
+		if !up.doCopy {
+			buf = uploadPool.Get()
+		}
+		return buf
+	}
+	// Put an upload token and a buffer
+	putBuf := func(buf []byte) {
+		if !up.doCopy {
+			uploadPool.Put(buf)
+		}
+		up.f.putBuf(nil, true)
+	}
 	g.Go(func() error {
 		for part := int64(1); part <= up.parts; part++ {
 			// Get a block of memory from the pool and token which limits concurrency.
-			buf := up.f.getBuf(up.doCopy)
+			buf := getBuf()
 
 			// Fail fast, in case an errgroup managed function returns an error
 			// gCtx is cancelled. There is no point in uploading all the other parts.
 			if gCtx.Err() != nil {
-				up.f.putBuf(buf, up.doCopy)
+				putBuf(buf)
 				return nil
 			}
 
@@ -454,14 +484,14 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 				buf = buf[:reqSize]
 				_, err = io.ReadFull(up.in, buf)
 				if err != nil {
-					up.f.putBuf(buf, up.doCopy)
+					putBuf(buf)
 					return err
 				}
 			}
 
 			part := part // for the closure
 			g.Go(func() (err error) {
-				defer up.f.putBuf(buf, up.doCopy)
+				defer putBuf(buf)
 				if !up.doCopy {
 					err = up.transferChunk(gCtx, part, buf)
 				} else {

backend/box/api/types.go

@@ -14,7 +14,7 @@ const (
 	timeFormat = `"` + time.RFC3339 + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // box API, by using RFC3339
 type Time time.Time
 

backend/box/box.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -27,6 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/golang-jwt/jwt/v4"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -45,7 +46,6 @@ import (
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/jws"
 )
 
 const (
@@ -76,6 +76,11 @@ var (
 	}
 )
 
+type boxCustomClaims struct {
+	jwt.StandardClaims
+	BoxSubType string `json:"box_sub_type,omitempty"`
+}
+
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -102,16 +107,18 @@ func init() {
 			return nil, nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
-			Name:     "root_folder_id",
-			Help:     "Fill in for rclone to use a non root folder as its starting point.",
-			Default:  "0",
-			Advanced: true,
+			Name:      "root_folder_id",
+			Help:      "Fill in for rclone to use a non root folder as its starting point.",
+			Default:   "0",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "box_config_file",
 			Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp,
 		}, {
-			Name: "access_token",
-			Help: "Box App Primary Access Token\n\nLeave blank normally.",
+			Name:      "access_token",
+			Help:      "Box App Primary Access Token\n\nLeave blank normally.",
+			Sensitive: true,
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -178,12 +185,12 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
 	client := fshttp.NewClient(ctx)
-	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+	err = jwtutil.Config("box", name, tokenURL, *claims, signingHeaders, queryParams, privateKey, m, client)
 	return err
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := ioutil.ReadFile(configFile)
+	file, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}
@@ -194,34 +201,31 @@ func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
 	return boxConfig, nil
 }
 
-func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) {
+func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomClaims, err error) {
 	val, err := jwtutil.RandomHex(20)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to generate random string for jti: %w", err)
 	}
 
-	claims = &jws.ClaimSet{
-		Iss: boxConfig.BoxAppSettings.ClientID,
-		Sub: boxConfig.EnterpriseID,
-		Aud: tokenURL,
-		Exp: time.Now().Add(time.Second * 45).Unix(),
-		PrivateClaims: map[string]interface{}{
-			"box_sub_type": boxSubType,
-			"aud":          tokenURL,
-			"jti":          val,
+	claims = &boxCustomClaims{
+		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
+		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
+		StandardClaims: jwt.StandardClaims{
+			Id:        val,
+			Issuer:    boxConfig.BoxAppSettings.ClientID,
+			Subject:   boxConfig.EnterpriseID,
+			Audience:  tokenURL,
+			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
 		},
+		BoxSubType: boxSubType,
 	}
-
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header {
-	signingHeaders := &jws.Header{
-		Algorithm: "RS256",
-		Typ:       "JWT",
-		KeyID:     boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
+func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
+	signingHeaders := map[string]interface{}{
+		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
-
 	return signingHeaders
 }
 
@@ -266,7 +270,7 @@ type Fs struct {
 	root         string                // the path we are working on
 	opt          Options               // parsed options
 	features     *fs.Features          // optional features
-	srv          *rest.Client          // the connection to the one drive server
+	srv          *rest.Client          // the connection to the server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
 	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
@@ -692,7 +696,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -752,7 +756,7 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -792,9 +796,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // PutUnchecked the object into the container
 //
-// This will produce an error if the object already exists
+// This will produce an error if the object already exists.
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -877,9 +881,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -897,7 +901,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	srcPath := srcObj.fs.rootSlash() + srcObj.remote
 	dstPath := f.rootSlash() + remote
-	if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
+	if strings.EqualFold(srcPath, dstPath) {
 		return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
 	}
 
@@ -995,9 +999,9 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1235,7 +1239,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1346,9 +1349,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
-// The new object may have been created if an error is returned
+// The new object may have been created if an error is returned.
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	if o.fs.tokenRenewer != nil {
 		o.fs.tokenRenewer.Start()

backend/cache/cache.go

@@ -1,6 +1,7 @@
 //go:build !plan9 && !js
 // +build !plan9,!js
 
+// Package cache implements a virtual provider to cache existing remotes.
 package cache
 
 import (
@@ -75,17 +76,19 @@ func init() {
 			Name: "plex_url",
 			Help: "The URL of the Plex server.",
 		}, {
-			Name: "plex_username",
-			Help: "The username of the Plex user.",
+			Name:      "plex_username",
+			Help:      "The username of the Plex user.",
+			Sensitive: true,
 		}, {
 			Name:       "plex_password",
 			Help:       "The password of the Plex user.",
 			IsPassword: true,
 		}, {
-			Name:     "plex_token",
-			Help:     "The plex token for authentication - auto set normally.",
-			Hide:     fs.OptionHideBoth,
-			Advanced: true,
+			Name:      "plex_token",
+			Help:      "The plex token for authentication - auto set normally.",
+			Hide:      fs.OptionHideBoth,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "plex_insecure",
 			Help:     "Skip all certificate verification when connecting to the Plex server.",
@@ -1037,7 +1040,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		}
 		fs.Debugf(dir, "list: remove entry: %v", entryRemote)
 	}
-	entries = nil
+	entries = nil //nolint:ineffassign
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
 	var batchDirectories []*Directory
@@ -1128,7 +1131,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				case fs.Directory:
 					_ = f.cache.AddDir(DirectoryFromOriginal(ctx, f, o))
 				default:
-					return fmt.Errorf("Unknown object type %T", entry)
+					return fmt.Errorf("unknown object type %T", entry)
 				}
 			}
 
@@ -1747,7 +1750,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1786,7 +1789,7 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
 	}
 }
 
-// StopBackgroundRunners will signall all the runners to stop their work
+// StopBackgroundRunners will signal all the runners to stop their work
 // can be triggered from a terminate signal or from testing between runs
 func (f *Fs) StopBackgroundRunners() {
 	f.cleanupChan <- false

backend/cache/cache_internal_test.go

@@ -11,7 +11,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -102,14 +101,12 @@ func TestMain(m *testing.M) {
 
 func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	id := fmt.Sprintf("tilrair%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	// Instantiate inner fs
 	innerFolder := "inner"
 	runInstance.mkdir(t, rootFs, innerFolder)
-	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs2, boltDb2)
+	rootFs2, _ := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil)
 
 	runInstance.writeObjectString(t, rootFs2, "one", "content")
 	listRoot, err := runInstance.list(t, rootFs, "")
@@ -167,7 +164,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -226,8 +223,7 @@ func TestInternalVfsCache(t *testing.T) {
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -259,8 +255,7 @@ func TestInternalObjWrapFsFound(t *testing.T) {
 
 func TestInternalObjNotFound(t *testing.T) {
 	id := fmt.Sprintf("tionf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
@@ -270,8 +265,7 @@ func TestInternalObjNotFound(t *testing.T) {
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -298,8 +292,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	// write the object
 	runInstance.writeRemoteString(t, rootFs, "one", "one content")
@@ -317,8 +310,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	var err error
 
 	// create some rand test data
@@ -347,8 +339,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -378,8 +369,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -405,8 +395,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -460,8 +449,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 
 func TestInternalMoveWithNotify(t *testing.T) {
 	id := fmt.Sprintf("timwn%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -547,8 +535,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
 
 func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	id := fmt.Sprintf("tincep%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -634,8 +621,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -667,8 +653,7 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"writes": "true"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -689,8 +674,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("timcsr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"workers": "1"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -725,8 +709,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
@@ -763,9 +746,7 @@ func TestInternalBug2117(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
-		map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
 
 	if runInstance.rootIsCrypt {
 		t.Skipf("skipping crypt")
@@ -841,7 +822,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
+		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -866,7 +847,7 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 	return enc
 }
 
-func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) {
+func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
 	for _, s := range config.FileSections() {
@@ -959,10 +940,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		runInstance.cleanupFs(t, f)
+	})
+
 	return f, boltDb
 }
 
-func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
+func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
 	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
@@ -984,7 +970,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := ioutil.TempFile("", "rclonecache-tempfile")
+	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {
@@ -1112,27 +1098,6 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
-func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = in.Close()
-	}()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = out.Close()
-	}()
-
-	_, err = io.Copy(out, in)
-	return err
-}
-
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 

backend/cache/cache_test.go

@@ -19,7 +19,7 @@ func TestIntegration(t *testing.T) {
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
 		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
 		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_upload_test.go

@@ -21,10 +21,8 @@ import (
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
-		nil,
+	runInstance.newCacheFs(t, remoteName, id, false, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -63,9 +61,7 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -73,19 +69,15 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -119,10 +111,8 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -162,21 +152,19 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
 	minSize := 5242880
 	maxSize := 10485760
 	totalFiles := 10
-	rand.Seed(time.Now().Unix())
+	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
 
 	lastFile := ""
 	for i := 0; i < totalFiles; i++ {
-		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"
 		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
@@ -213,9 +201,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -343,9 +329,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 

backend/cache/plex.go

@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = ioutil.ReadAll(resp.Body)
+							data, err = io.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}
@@ -213,7 +213,7 @@ func (p *plexConnector) authenticate() error {
 	var data map[string]interface{}
 	err = json.NewDecoder(resp.Body).Decode(&data)
 	if err != nil {
-		return fmt.Errorf("failed to obtain token: %v", err)
+		return fmt.Errorf("failed to obtain token: %w", err)
 	}
 	tokenGen, ok := get(data, "user", "authToken")
 	if !ok {

backend/cache/storage_memory.go

@@ -76,10 +76,7 @@ func (m *Memory) CleanChunksByAge(chunkAge time.Duration) {
 
 // CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk
 func (m *Memory) CleanChunksByNeed(offset int64) {
-	var items map[string]cache.Item
-
-	items = m.db.Items()
-	for key := range items {
+	for key := range m.db.Items() {
 		sepIdx := strings.LastIndex(key, "-")
 		keyOffset, err := strconv.ParseInt(key[sepIdx+1:], 10, 64)
 		if err != nil {

backend/cache/storage_persistent.go

@@ -9,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -250,7 +249,7 @@ func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)
 		if val != nil {
 			err := json.Unmarshal(val, cachedDir)
 			if err != nil {
-				return fmt.Errorf("error during unmarshalling obj: %v", err)
+				return fmt.Errorf("error during unmarshalling obj: %w", err)
 			}
 		} else {
 			return fmt.Errorf("missing cached dir: %v", cachedDir)
@@ -456,10 +455,7 @@ func (b *Persistent) HasEntry(remote string) bool {
 
 		return fmt.Errorf("couldn't find object (%v)", remote)
 	})
-	if err == nil {
-		return true
-	}
-	return false
+	return err == nil
 }
 
 // HasChunk confirms the existence of a single chunk of an object
@@ -476,7 +472,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := ioutil.ReadFile(fp)
+	data, err := os.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -489,7 +485,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := ioutil.WriteFile(filePath, data, os.ModePerm)
+	err := os.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}
@@ -554,7 +550,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		dataTsBucket := tx.Bucket([]byte(DataTsBucket))
 		if dataTsBucket == nil {
-			return fmt.Errorf("Couldn't open (%v) bucket", DataTsBucket)
+			return fmt.Errorf("couldn't open (%v) bucket", DataTsBucket)
 		}
 		// iterate through ts
 		c := dataTsBucket.Cursor()
@@ -904,16 +900,16 @@ func (b *Persistent) rollbackPendingUpload(remote string) error {
 		v := bucket.Get([]byte(remote))
 		err = json.Unmarshal(v, tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload (%v) not found %v", remote, err)
+			return fmt.Errorf("pending upload (%v) not found: %w", remote, err)
 		}
 		tempObj.Started = false
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		return nil
 	})
@@ -969,11 +965,11 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 		}
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 
 		return nil

backend/chunker/chunker.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -32,7 +31,6 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 )
 
-//
 // Chunker's composite files have one or more chunks
 // and optional metadata object. If it's present,
 // meta object is named after the original file.
@@ -65,7 +63,7 @@ import (
 // length of 13 decimals it makes a 7-digit base-36 number.
 //
 // When transactions is set to the norename style, data chunks will
-// keep their temporary chunk names (with the transacion identifier
+// keep their temporary chunk names (with the transaction identifier
 // suffix). To distinguish them from temporary chunks, the txn field
 // of the metadata file is set to match the transaction identifier of
 // the data chunks.
@@ -79,7 +77,6 @@ import (
 // Metadata format v1 does not define any control chunk types,
 // they are currently ignored aka reserved.
 // In future they can be used to implement resumable uploads etc.
-//
 const (
 	ctrlTypeRegStr   = `[a-z][a-z0-9]{2,6}`
 	tempSuffixFormat = `_%04s`
@@ -515,7 +512,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 
 	strRegex := regexp.QuoteMeta(pattern)
 	strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
-	strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
+	strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
 	strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
 	f.nameRegexp = regexp.MustCompile(strRegex)
 
@@ -524,7 +521,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 	if numDigits > 1 {
 		fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
 	}
-	strFmt := strings.Replace(pattern, "%", "%%", -1)
+	strFmt := strings.ReplaceAll(pattern, "%", "%%")
 	strFmt = strings.Replace(strFmt, "*", "%s", 1)
 	f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
 	f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
@@ -542,7 +539,6 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 //
 // xactID is a transaction identifier. Empty xactID denotes active chunk,
 // otherwise temporary chunk name is produced.
-//
 func (f *Fs) makeChunkName(filePath string, chunkNo int, ctrlType, xactID string) string {
 	dir, parentName := path.Split(filePath)
 	var name, tempSuffix string
@@ -708,7 +704,6 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
 // directory together with dead chunks.
 // In future a flag named like `--chunker-list-hidden` may be added to
 // rclone that will tell List to reveal hidden chunks.
-//
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	entries, err = f.base.List(ctx, dir)
 	if err != nil {
@@ -868,7 +863,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 // Note that chunker prefers analyzing file names rather than reading
 // the content of meta object assuming that directory scans are fast
 // but opening even a small file can be slow on some backends.
-//
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.scanObject(ctx, remote, false)
 }
@@ -1043,7 +1037,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := ioutil.ReadAll(reader)
+	metadata, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1084,7 +1078,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 
 // readXactID returns the transaction ID stored in the passed metadata object
 func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
-	// if xactID has already been read and cahced return it now
+	// if xactID has already been read and cached return it now
 	if o.xIDCached {
 		return o.xactID, nil
 	}
@@ -1102,7 +1096,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadAll(reader)
+	data, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err
@@ -1586,7 +1580,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 // This command will chain to `purge` from wrapped remote.
 // As a result it removes not only composite chunker files with their
 // active chunks but also all hidden temporary chunks in the directory.
-//
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	do := f.base.Features().Purge
 	if do == nil {
@@ -1628,7 +1621,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Unsupported control chunks will get re-picked by a more recent
 // rclone version with unexpected results. This can be helped by
 // the `delete hidden` flag above or at least the user has been warned.
-//
 func (o *Object) Remove(ctx context.Context) (err error) {
 	if err := o.f.forbidChunk(o, o.Remote()); err != nil {
 		// operations.Move can still call Remove if chunker's Move refuses
@@ -1804,9 +1796,9 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string)
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1825,9 +1817,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1895,7 +1887,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.base.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1904,7 +1896,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.base.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -2125,7 +2117,6 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
 // file, then tries to read it from metadata. This in theory
 // handles the unusual case when a small file has been tampered
 // on the level of wrapped remote but chunker is unaware of that.
-//
 func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) {
 	if err := o.readMetadata(ctx); err != nil {
 		return "", err // valid metadata is required to get hash, abort
@@ -2414,7 +2405,6 @@ type metaSimpleJSON struct {
 // - for files larger than chunk size
 // - if file contents can be mistaken as meta object
 // - if consistent hashing is On but wrapped remote can't provide given hash
-//
 func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) {
 	version := metadataVersion
 	if xactID == "" && version == 2 {
@@ -2447,7 +2437,6 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1,
 // New format will have a higher version number and cannot be correctly
 // handled by current implementation.
 // The version check below will then explicitly ask user to upgrade rclone.
-//
 func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) {
 	// Be strict about JSON format
 	// to reduce possibility that a random small file resembles metadata.

backend/chunker/chunker_internal_test.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -59,7 +59,7 @@ var mtime1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
 
 func testPutFile(ctx context.Context, t *testing.T, f fs.Fs, name, contents, message string, check bool) fs.Object {
 	item := fstest.Item{Path: name, ModTime: mtime1}
-	_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
+	obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
 	assert.NotNil(t, obj, message)
 	return obj
 }
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -440,7 +440,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 	checkSmallFile := func(name, contents string) {
 		filename := path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
+		put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
 		assert.NotNil(t, put)
 		checkSmallFileInternals(put)
 		checkContents(put, contents)
@@ -489,7 +489,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 
 	newFile := func(name string) fs.Object {
 		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		return obj
 	}
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
+		chunkContents, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
+		_, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -599,7 +599,7 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
 		filename = path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
 			txnID = chunkObj.xactID
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
+			data, err := io.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -716,7 +716,7 @@ func testFutureProof(t *testing.T, f *Fs) {
 			name = f.makeChunkName(name, part-1, "", "")
 		}
 		item := fstest.Item{Path: name, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
+		obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
 		assert.NotNil(t, obj, msg)
 	}
 
@@ -758,8 +758,8 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
+	in := io.NopCloser(bytes.NewBufferString("abc"))
+	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
@@ -790,7 +790,7 @@ func testBackwardsCompatibility(t *testing.T, f *Fs) {
 	newFile := func(f fs.Fs, name string) (fs.Object, string) {
 		filename := path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		return obj, filename
 	}
@@ -844,7 +844,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	item := fstest.Item{Path: "movefile", ModTime: modTime}
 	contents := "abcdef"
-	_, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
+	file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
 
 	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
 	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
+	data, err := io.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()

backend/chunker/chunker_test.go

@@ -35,6 +35,7 @@ func TestIntegration(t *testing.T) {
 			"MimeType",
 			"GetTier",
 			"SetTier",
+			"Metadata",
 		},
 		UnimplementableFsMethods: []string{
 			"PublicLink",
@@ -53,6 +54,7 @@ func TestIntegration(t *testing.T) {
 			{Name: name, Key: "type", Value: "chunker"},
 			{Name: name, Key: "remote", Value: tempDir},
 		}
+		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
 }

backend/combine/combine_internal_test.go

@@ -0,0 +1,94 @@
+package combine
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAdjustmentDo(t *testing.T) {
+	for _, test := range []struct {
+		root       string
+		mountpoint string
+		in         string
+		want       string
+		wantErr    error
+	}{
+		{
+			root:       "",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "mountpoint/path/to/file.txt",
+		},
+		{
+			root:       "mountpoint",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "wrongpath/to/file.txt",
+			want:       "",
+			wantErr:    errNotUnderRoot,
+		},
+	} {
+		what := fmt.Sprintf("%+v", test)
+		a := newAdjustment(test.root, test.mountpoint)
+		got, gotErr := a.do(test.in)
+		assert.Equal(t, test.wantErr, gotErr)
+		assert.Equal(t, test.want, got, what)
+	}
+
+}
+
+func TestAdjustmentUndo(t *testing.T) {
+	for _, test := range []struct {
+		root       string
+		mountpoint string
+		in         string
+		want       string
+		wantErr    error
+	}{
+		{
+			root:       "",
+			mountpoint: "mountpoint",
+			in:         "mountpoint/path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "wrongmountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "to/file.txt",
+			want:       "",
+			wantErr:    errNotUnderRoot,
+		},
+	} {
+		what := fmt.Sprintf("%+v", test)
+		a := newAdjustment(test.root, test.mountpoint)
+		got, gotErr := a.undo(test.in)
+		assert.Equal(t, test.wantErr, gotErr)
+		assert.Equal(t, test.want, got, what)
+	}
+
+}

backend/combine/combine_test.go

@@ -0,0 +1,92 @@
+// Test Combine filesystem interface
+package combine_test
+
+import (
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	_ "github.com/rclone/rclone/backend/memory"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+var (
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect"}
+	unimplementableObjectMethods = []string{}
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	if *fstest.RemoteName == "" {
+		t.Skip("Skipping as -remote not set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:                   *fstest.RemoteName,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+func TestLocal(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	dirs := MakeTestDirs(t, 3)
+	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=" + dirs[2]
+	name := "TestCombineLocal"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+func TestMemory(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	upstreams := "dir1=:memory:dir1 dir2=:memory:dir2 dir3=:memory:dir3"
+	name := "TestCombineMemory"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+func TestMixed(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	dirs := MakeTestDirs(t, 2)
+	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=:memory:dir3"
+	name := "TestCombineMixed"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+// MakeTestDirs makes directories in /tmp for testing
+func MakeTestDirs(t *testing.T, n int) (dirs []string) {
+	for i := 1; i <= n; i++ {
+		dir := t.TempDir()
+		dirs = append(dirs, dir)
+	}
+	return dirs
+}

backend/compress/compress.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -29,6 +28,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 )
@@ -53,7 +53,7 @@ const (
 	Gzip         = 2
 )
 
-var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
+var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`)
 
 // Register with Fs
 func init() {
@@ -70,6 +70,9 @@ func init() {
 		Name:        "compress",
 		Description: "Compress a remote",
 		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to compress.",
@@ -87,7 +90,7 @@ Generally -1 (default, equivalent to 5) is recommended.
 Levels 1 to 9 increase compression at the cost of speed. Going past 6 
 generally offers very little return.
 
-Level -2 uses Huffmann encoding only. Only use if you know what you
+Level -2 uses Huffman encoding only. Only use if you know what you
 are doing.
 Level 0 turns off compression.`,
 			Default:  sgzip.DefaultCompression,
@@ -127,7 +130,7 @@ type Fs struct {
 	features *fs.Features // optional features
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -180,6 +183,10 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		SetTier:                 true,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -222,7 +229,7 @@ func processFileName(compressedFileName string) (origFileName string, extension
 	// Separate the filename and size from the extension
 	extensionPos := strings.LastIndex(compressedFileName, ".")
 	if extensionPos == -1 {
-		return "", "", 0, errors.New("File name has no extension")
+		return "", "", 0, errors.New("file name has no extension")
 	}
 	extension = compressedFileName[extensionPos:]
 	nameWithSize := compressedFileName[:extensionPos]
@@ -231,11 +238,11 @@ func processFileName(compressedFileName string) (origFileName string, extension
 	}
 	match := nameRegexp.FindStringSubmatch(nameWithSize)
 	if match == nil || len(match) != 3 {
-		return "", "", 0, errors.New("Invalid filename")
+		return "", "", 0, errors.New("invalid filename")
 	}
 	size, err := base64ToInt64(match[2])
 	if err != nil {
-		return "", "", 0, errors.New("Could not decode size")
+		return "", "", 0, errors.New("could not decode size")
 	}
 	return match[1], gzFileExt, size, nil
 }
@@ -304,7 +311,7 @@ func (f *Fs) processEntries(entries fs.DirEntries) (newEntries fs.DirEntries, er
 		case fs.Directory:
 			f.addDir(&newEntries, x)
 		default:
-			return nil, fmt.Errorf("Unknown object type %T", entry)
+			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
 	}
 	return newEntries, nil
@@ -361,13 +368,16 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	meta := readMetadata(ctx, mo)
-	if meta == nil {
-		return nil, errors.New("error decoding metadata")
+	meta, err := readMetadata(ctx, mo)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding metadata: %w", err)
 	}
 	// Create our Object
 	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
-	return f.newObject(o, mo, meta), err
+	if err != nil {
+		return nil, err
+	}
+	return f.newObject(o, mo, meta), nil
 }
 
 // checkCompressAndType checks if an object is compressible and determines it's mime type
@@ -445,7 +455,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 		return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
 	}
 
-	// Need to include what we allready read
+	// Need to include what we already read
 	in = &ReadCloserWrapper{
 		Reader: io.MultiReader(bytes.NewReader(buf), in),
 		Closer: in,
@@ -458,7 +468,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := ioutil.TempFile("", "rclone-press-")
+	tempFile, err := os.CreateTemp("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -466,10 +476,10 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 		_ = os.Remove(tempFile.Name())
 	}()
 	if err != nil {
-		return nil, fmt.Errorf("Failed to create temporary local FS to spool file: %w", err)
+		return nil, fmt.Errorf("failed to create temporary local FS to spool file: %w", err)
 	}
 	if _, err = io.Copy(tempFile, in); err != nil {
-		return nil, fmt.Errorf("Failed to write temporary local file: %w", err)
+		return nil, fmt.Errorf("failed to write temporary local file: %w", err)
 	}
 	if _, err = tempFile.Seek(0, 0); err != nil {
 		return nil, err
@@ -536,8 +546,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)
@@ -671,7 +681,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		}
 		return nil, err
 	}
-	return f.newObject(dataObject, mo, meta), err
+	return f.newObject(dataObject, mo, meta), nil
 }
 
 // Put in to the remote path with the modTime given of the given size
@@ -720,23 +730,23 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	if found && (oldObj.(*Object).meta.Mode != Uncompressed || compressible) {
 		err = oldObj.(*Object).Object.Remove(ctx)
 		if err != nil {
-			return nil, fmt.Errorf("Could remove original object: %w", err)
+			return nil, fmt.Errorf("couldn't remove original object: %w", err)
 		}
 	}
 
 	// If our new object is compressed we have to rename it with the correct size.
-	// Uncompressed objects don't store the size in the name so we they'll allready have the correct name.
+	// Uncompressed objects don't store the size in the name so we they'll already have the correct name.
 	if compressible {
 		wrapObj, err := operations.Move(ctx, f.Fs, nil, f.dataName(src.Remote(), newObj.size, compressible), newObj.Object)
 		if err != nil {
-			return nil, fmt.Errorf("Couldn't rename streamed Object.: %w", err)
+			return nil, fmt.Errorf("couldn't rename streamed object: %w", err)
 		}
 		newObj.Object = wrapObj
 	}
 	return newObj, nil
 }
 
-// Temporarely disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
+// Temporarily disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
 // will break stuff. Right no I can't think of a way to make this work.
 
 // PutUnchecked uploads the object
@@ -779,9 +789,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -829,9 +839,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -906,7 +916,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.Fs.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp: not supported by underlying remote")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -915,7 +925,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("can't About: not supported by underlying remote")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1034,24 +1044,19 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim
 }
 
 // This function will read the metadata from a metadata object.
-func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) {
+func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
 	// Open our meradata object
 	rc, err := mo.Open(ctx)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-	defer func() {
-		err := rc.Close()
-		if err != nil {
-			fs.Errorf(mo, "Error closing object: %v", err)
-		}
-	}()
+	defer fs.CheckClose(rc, &err)
 	jr := json.NewDecoder(rc)
 	meta = new(ObjectMetadata)
 	if err = jr.Decode(meta); err != nil {
-		return nil
+		return nil, err
 	}
-	return meta
+	return meta, nil
 }
 
 // Remove removes this object
@@ -1096,6 +1101,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	origName := o.Remote()
 	if o.meta.Mode != Uncompressed || compressible {
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType)
+		if err != nil {
+			return err
+		}
 		if newObject.Object.Remote() != o.Object.Remote() {
 			if removeErr := o.Object.Remove(ctx); removeErr != nil {
 				return removeErr
@@ -1109,9 +1117,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		// If we are, just update the object and metadata
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType)
-	}
-	if err != nil {
-		return err
+		if err != nil {
+			return err
+		}
 	}
 	// Update object metadata and return
 	o.Object = newObject.Object
@@ -1122,6 +1130,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified.
 func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object {
+	if o == nil {
+		log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1134,6 +1145,9 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object
 
 // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand.
 func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object {
+	if o == nil {
+		log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1161,7 +1175,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) {
 		return err
 	}
 	if o.meta == nil {
-		o.meta = readMetadata(ctx, o.mo)
+		o.meta, err = readMetadata(ctx, o.mo)
 	}
 	return err
 }
@@ -1214,6 +1228,21 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return o.meta.MimeType
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	err := o.loadMetadataIfNotLoaded(ctx)
+	if err != nil {
+		return nil, err
+	}
+	do, ok := o.mo.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
@@ -1360,6 +1389,51 @@ func (o *ObjectInfo) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	return "", nil // cannot know the checksum
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *ObjectInfo) ID() string {
+	do, ok := o.src.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+func (o *ObjectInfo) MimeType(ctx context.Context) string {
+	do, ok := o.src.(fs.MimeTyper)
+	if !ok {
+		return ""
+	}
+	return do.MimeType(ctx)
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *ObjectInfo) UnWrap() fs.Object {
+	return fs.UnWrapObjectInfo(o.src)
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.src.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *ObjectInfo) GetTier() string {
+	do, ok := o.src.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
 // ID returns the ID of the Object if known, or "" if not
 func (o *Object) ID() string {
 	do, ok := o.Object.(fs.IDer)
@@ -1412,11 +1486,6 @@ var (
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.FullObjectInfo  = (*ObjectInfo)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/compress/compress_test.go

@@ -61,5 +61,6 @@ func TestRemoteGzip(t *testing.T) {
 			{Name: name, Key: "remote", Value: tempdir},
 			{Name: name, Key: "compression_mode", Value: "gzip"},
 		},
+		QuickTestOK: true,
 	})
 }

backend/crypt/cipher.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
@@ -37,7 +38,6 @@ const (
 	blockHeaderSize     = secretbox.Overhead
 	blockDataSize       = 64 * 1024
 	blockSize           = blockHeaderSize + blockDataSize
-	encryptedSuffix     = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
 )
 
 // Errors returned by cipher
@@ -53,8 +53,9 @@ var (
 	ErrorEncryptedBadBlock       = errors.New("failed to authenticate decrypted block - bad password?")
 	ErrorBadBase32Encoding       = errors.New("bad base32 filename encoding")
 	ErrorFileClosed              = errors.New("file already closed")
-	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
+	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - does not match suffix")
 	ErrorBadSeek                 = errors.New("Seek beyond end of file")
+	ErrorSuffixMissingDot        = errors.New("suffix config setting should include a '.'")
 	defaultSalt                  = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
 	obfuscQuoteRune              = '!'
 )
@@ -96,7 +97,7 @@ func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
 	case "obfuscate":
 		mode = NameEncryptionObfuscated
 	default:
-		err = fmt.Errorf("Unknown file name encryption mode %q", s)
+		err = fmt.Errorf("unknown file name encryption mode %q", s)
 	}
 	return mode, err
 }
@@ -127,11 +128,11 @@ type fileNameEncoding interface {
 // RFC4648
 //
 // The standard encoding is modified in two ways
-//  * it becomes lower case (no-one likes upper case filenames!)
-//  * we strip the padding character `=`
+//   - it becomes lower case (no-one likes upper case filenames!)
+//   - we strip the padding character `=`
 type caseInsensitiveBase32Encoding struct{}
 
-// EncodeToString encodes a strign using the modified version of
+// EncodeToString encodes a string using the modified version of
 // base32 encoding.
 func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
 	encoded := base32.HexEncoding.EncodeToString(src)
@@ -162,34 +163,37 @@ func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
 	case "base32768":
 		enc = base32768.SafeEncoding
 	default:
-		err = fmt.Errorf("Unknown file name encoding mode %q", s)
+		err = fmt.Errorf("unknown file name encoding mode %q", s)
 	}
 	return enc, err
 }
 
 // Cipher defines an encoding and decoding cipher for the crypt backend
 type Cipher struct {
-	dataKey        [32]byte                  // Key for secretbox
-	nameKey        [32]byte                  // 16,24 or 32 bytes
-	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
-	block          gocipher.Block
-	mode           NameEncryptionMode
-	fileNameEnc    fileNameEncoding
-	buffers        sync.Pool // encrypt/decrypt buffers
-	cryptoRand     io.Reader // read crypto random numbers from here
-	dirNameEncrypt bool
+	dataKey         [32]byte                  // Key for secretbox
+	nameKey         [32]byte                  // 16,24 or 32 bytes
+	nameTweak       [nameCipherBlockSize]byte // used to tweak the name crypto
+	block           gocipher.Block
+	mode            NameEncryptionMode
+	fileNameEnc     fileNameEncoding
+	buffers         sync.Pool // encrypt/decrypt buffers
+	cryptoRand      io.Reader // read crypto random numbers from here
+	dirNameEncrypt  bool
+	passBadBlocks   bool // if set passed bad blocks as zeroed blocks
+	encryptedSuffix string
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
 func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
-		mode:           mode,
-		fileNameEnc:    enc,
-		cryptoRand:     rand.Reader,
-		dirNameEncrypt: dirNameEncrypt,
+		mode:            mode,
+		fileNameEnc:     enc,
+		cryptoRand:      rand.Reader,
+		dirNameEncrypt:  dirNameEncrypt,
+		encryptedSuffix: ".bin",
 	}
 	c.buffers.New = func() interface{} {
-		return make([]byte, blockSize)
+		return new([blockSize]byte)
 	}
 	err := c.Key(password, salt)
 	if err != nil {
@@ -198,11 +202,29 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
+// setEncryptedSuffix set suffix, or an empty string
+func (c *Cipher) setEncryptedSuffix(suffix string) {
+	if strings.EqualFold(suffix, "none") {
+		c.encryptedSuffix = ""
+		return
+	}
+	if !strings.HasPrefix(suffix, ".") {
+		fs.Errorf(nil, "crypt: bad suffix: %v", ErrorSuffixMissingDot)
+		suffix = "." + suffix
+	}
+	c.encryptedSuffix = suffix
+}
+
+// Call to set bad block pass through
+func (c *Cipher) setPassBadBlocks(passBadBlocks bool) {
+	c.passBadBlocks = passBadBlocks
+}
+
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
 // If salt is "" we use a fixed salt just to make attackers lives
-// slighty harder than using no salt.
+// slightly harder than using no salt.
 //
 // Note that empty password makes all 0x00 keys which is used in the
 // tests.
@@ -230,21 +252,18 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() []byte {
-	return c.buffers.Get().([]byte)
+func (c *Cipher) getBlock() *[blockSize]byte {
+	return c.buffers.Get().(*[blockSize]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf []byte) {
-	if len(buf) != blockSize {
-		panic("bad blocksize returned to pool")
-	}
+func (c *Cipher) putBlock(buf *[blockSize]byte) {
 	c.buffers.Put(buf)
 }
 
 // encryptSegment encrypts a path segment
 //
-// This uses EME with AES
+// This uses EME with AES.
 //
 // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
@@ -254,8 +273,8 @@ func (c *Cipher) putBlock(buf []byte) {
 // same filename must encrypt to the same thing.
 //
 // This means that
-//  * filenames with the same name will encrypt the same
-//  * filenames which start the same won't have a common prefix
+//   - filenames with the same name will encrypt the same
+//   - filenames which start the same won't have a common prefix
 func (c *Cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
@@ -508,7 +527,7 @@ func (c *Cipher) encryptFileName(in string) string {
 // EncryptFileName encrypts a file path
 func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
-		return in + encryptedSuffix
+		return in + c.encryptedSuffix
 	}
 	return c.encryptFileName(in)
 }
@@ -568,8 +587,8 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 // DecryptFileName decrypts a file path
 func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
-		remainingLength := len(in) - len(encryptedSuffix)
-		if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
+		remainingLength := len(in) - len(c.encryptedSuffix)
+		if remainingLength == 0 || !strings.HasSuffix(in, c.encryptedSuffix) {
 			return "", ErrorNotAnEncryptedFile
 		}
 		decrypted := in[:remainingLength]
@@ -609,7 +628,7 @@ func (n *nonce) pointer() *[fileNonceSize]byte {
 // fromReader fills the nonce from an io.Reader - normally the OSes
 // crypto random number generator
 func (n *nonce) fromReader(in io.Reader) error {
-	read, err := io.ReadFull(in, (*n)[:])
+	read, err := readers.ReadFill(in, (*n)[:])
 	if read != fileNonceSize {
 		return fmt.Errorf("short read of nonce: %w", err)
 	}
@@ -664,8 +683,8 @@ type encrypter struct {
 	in       io.Reader
 	c        *Cipher
 	nonce    nonce
-	buf      []byte
-	readBuf  []byte
+	buf      *[blockSize]byte
+	readBuf  *[blockSize]byte
 	bufIndex int
 	bufSize  int
 	err      error
@@ -690,9 +709,9 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 		}
 	}
 	// Copy magic into buffer
-	copy(fh.buf, fileMagicBytes)
+	copy((*fh.buf)[:], fileMagicBytes)
 	// Copy nonce into buffer
-	copy(fh.buf[fileMagicSize:], fh.nonce[:])
+	copy((*fh.buf)[fileMagicSize:], fh.nonce[:])
 	return fh, nil
 }
 
@@ -707,22 +726,20 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 	if fh.bufIndex >= fh.bufSize {
 		// Read data
 		// FIXME should overlap the reads with a go-routine and 2 buffers?
-		readBuf := fh.readBuf[:blockDataSize]
-		n, err = io.ReadFull(fh.in, readBuf)
+		readBuf := (*fh.readBuf)[:blockDataSize]
+		n, err = readers.ReadFill(fh.in, readBuf)
 		if n == 0 {
-			// err can't be nil since:
-			// n == len(buf) if and only if err == nil.
 			return fh.finish(err)
 		}
 		// possibly err != nil here, but we will process the
-		// data and the next call to ReadFull will return 0, err
+		// data and the next call to ReadFill will return 0, err
 		// Encrypt the block using the nonce
-		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		secretbox.Seal((*fh.buf)[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufSize])
 	fh.bufIndex += n
 	return n, nil
 }
@@ -763,8 +780,8 @@ type decrypter struct {
 	nonce        nonce
 	initialNonce nonce
 	c            *Cipher
-	buf          []byte
-	readBuf      []byte
+	buf          *[blockSize]byte
+	readBuf      *[blockSize]byte
 	bufIndex     int
 	bufSize      int
 	err          error
@@ -782,12 +799,12 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 		limit:   -1,
 	}
 	// Read file header (magic + nonce)
-	readBuf := fh.readBuf[:fileHeaderSize]
-	_, err := io.ReadFull(fh.rc, readBuf)
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
+	readBuf := (*fh.readBuf)[:fileHeaderSize]
+	n, err := readers.ReadFill(fh.rc, readBuf)
+	if n < fileHeaderSize && err == io.EOF {
 		// This read from 0..fileHeaderSize-1 bytes
 		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
-	} else if err != nil {
+	} else if err != io.EOF && err != nil {
 		return nil, fh.finishAndClose(err)
 	}
 	// check the magic
@@ -845,10 +862,8 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 func (fh *decrypter) fillBuffer() (err error) {
 	// FIXME should overlap the reads with a go-routine and 2 buffers?
 	readBuf := fh.readBuf
-	n, err := io.ReadFull(fh.rc, readBuf)
+	n, err := readers.ReadFill(fh.rc, (*readBuf)[:])
 	if n == 0 {
-		// err can't be nil since:
-		// n == len(buf) if and only if err == nil.
 		return err
 	}
 	// possibly err != nil here, but we will process the data and
@@ -856,18 +871,25 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	// Check header + 1 byte exists
 	if n <= blockHeaderSize {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	_, ok := secretbox.Open((*fh.buf)[:0], (*readBuf)[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
-		return ErrorEncryptedBadBlock
+		if !fh.c.passBadBlocks {
+			return ErrorEncryptedBadBlock
+		}
+		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
+		// Zero out the bad block and continue
+		for i := range (*fh.buf)[:n] {
+			(*fh.buf)[i] = 0
+		}
 	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize
@@ -893,7 +915,7 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	if fh.limit >= 0 && fh.limit < int64(toCopy) {
 		toCopy = int(fh.limit)
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufIndex+toCopy])
 	fh.bufIndex += n
 	if fh.limit >= 0 {
 		fh.limit -= int64(n)
@@ -904,9 +926,8 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-// calculateUnderlying converts an (offset, limit) in a crypted file
-// into an (underlyingOffset, underlyingLimit) for the underlying
-// file.
+// calculateUnderlying converts an (offset, limit) in an encrypted file
+// into an (underlyingOffset, underlyingLimit) for the underlying file.
 //
 // It also returns number of bytes to discard after reading the first
 // block and number of blocks this is from the start so the nonce can
@@ -1085,7 +1106,7 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 
 // DecryptDataSeek decrypts the data stream from offset
 //
-// The open function must return a ReadCloser opened to the offset supplied
+// The open function must return a ReadCloser opened to the offset supplied.
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
 func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {

backend/crypt/cipher_test.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -28,14 +27,14 @@ func TestNewNameEncryptionMode(t *testing.T) {
 		{"off", NameEncryptionOff, ""},
 		{"standard", NameEncryptionStandard, ""},
 		{"obfuscate", NameEncryptionObfuscated, ""},
-		{"potato", NameEncryptionOff, "Unknown file name encryption mode \"potato\""},
+		{"potato", NameEncryptionOff, "unknown file name encryption mode \"potato\""},
 	} {
 		actual, actualErr := NewNameEncryptionMode(test.in)
 		assert.Equal(t, actual, test.expected)
 		if test.expectedErr == "" {
 			assert.NoError(t, actualErr)
 		} else {
-			assert.Error(t, actualErr, test.expectedErr)
+			assert.EqualError(t, actualErr, test.expectedErr)
 		}
 	}
 }
@@ -406,6 +405,13 @@ func TestNonStandardEncryptFileName(t *testing.T) {
 	// Off mode
 	c, _ := newCipher(NameEncryptionOff, "", "", true, nil)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
+	// Off mode with custom suffix
+	c, _ = newCipher(NameEncryptionOff, "", "", true, nil)
+	c.setEncryptedSuffix(".jpg")
+	assert.Equal(t, "1/12/123.jpg", c.EncryptFileName("1/12/123"))
+	// Off mode with empty suffix
+	c.setEncryptedSuffix("none")
+	assert.Equal(t, "1/12/123", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", true, nil)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
@@ -484,21 +490,27 @@ func TestNonStandardDecryptFileName(t *testing.T) {
 			in             string
 			expected       string
 			expectedErr    error
+			customSuffix   string
 		}{
-			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
-			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
-			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
-			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
-			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
-			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
-			{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
-			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
-			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
-			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
-			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
-			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil, ""},
+			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile, ""},
+			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile, ""},
+			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil, ""},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil, ""},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil, ""},
+			{NameEncryptionOff, true, "1/12/123.jpg", "1/12/123", nil, ".jpg"},
+			{NameEncryptionOff, true, "1/12/123", "1/12/123", nil, "none"},
+			{NameEncryptionObfuscated, true, "!.hello", "hello", nil, ""},
+			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile, ""},
+			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil, ""},
+			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil, ""},
+			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil, ""},
+			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil, ""},
 		} {
 			c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
+			if test.customSuffix != "" {
+				c.setEncryptedSuffix(test.customSuffix)
+			}
 			actual, actualErr := c.DecryptFileName(test.in)
 			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
 			assert.Equal(t, test.expected, actual, what)
@@ -727,7 +739,7 @@ func TestNonceFromReader(t *testing.T) {
 	assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
 	buf = bytes.NewBufferString("123456789abcdefghijklmn")
 	err = x.fromReader(buf)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 func TestNonceFromBuf(t *testing.T) {
@@ -1051,7 +1063,7 @@ func TestRandomSource(t *testing.T) {
 	_, _ = source.Read(buf)
 	sink = newRandomSource(1e8)
 	_, err = io.Copy(sink, source)
-	assert.Error(t, err, "Error in stream")
+	assert.EqualError(t, err, "Error in stream at 1")
 }
 
 type zeroes struct{}
@@ -1073,7 +1085,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1144,15 +1156,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := ioutil.ReadAll(encrypted)
+		out, err := io.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
+		decrypted, err := c.DecryptData(io.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = ioutil.ReadAll(decrypted)
+		out, err = io.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1168,13 +1180,13 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err := c.newEncrypter(z, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
-	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
+	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, (*fh.buf)[:32])
 
 	// Test error path
 	c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
@@ -1187,7 +1199,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1225,7 +1237,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
 		fh, err = c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedFileTooShort.Error())
+		assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
 		assert.Equal(t, 1, cd.closed)
 	}
 
@@ -1233,7 +1245,7 @@ func TestNewDecrypter(t *testing.T) {
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "potato")
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 1, cd.closed)
 
 	// bad magic
@@ -1244,7 +1256,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0copy))
 		fh, err := c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+		assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 		file0copy[i] ^= 0x1
 		assert.Equal(t, 1, cd.closed)
 	}
@@ -1257,12 +1269,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := ioutil.NopCloser(io.MultiReader(in1, in2))
+	in := io.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1274,14 +1286,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
+	plaintext, err := io.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := ioutil.ReadAll(encrypted)
+	ciphertext, err := io.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1300,7 +1312,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1490,14 +1502,16 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = ioutil.ReadAll(fh)
+		_, err = io.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
 			// This would normally produce an error *except* on the first block
 			expectedErr = nil
+		case i <= fileHeaderSize+blockHeaderSize:
+			expectedErr = ErrorEncryptedFileBadHeader
 		default:
-			expectedErr = io.ErrUnexpectedEOF
+			expectedErr = ErrorEncryptedBadBlock
 		}
 		if expectedErr != nil {
 			assert.EqualError(t, err, expectedErr.Error(), what)
@@ -1514,8 +1528,8 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = ioutil.ReadAll(fh)
-	assert.Error(t, err, "potato")
+	_, err = io.ReadAll(fh)
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
 	// Test corrupting the input
@@ -1524,17 +1538,28 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
-			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+			assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = ioutil.ReadAll(fh)
-			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
+			_, err = io.ReadAll(fh)
+			assert.EqualError(t, err, ErrorEncryptedBadBlock.Error())
 		}
 		file16copy[i] ^= 0xFF
 	}
+
+	// Test that we can corrupt a byte and read zeroes if
+	// passBadBlocks is set
+	copy(file16copy, file16)
+	file16copy[len(file16copy)-1] ^= 0xFF
+	c.passBadBlocks = true
+	fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
+	assert.NoError(t, err)
+	buf, err := io.ReadAll(fh)
+	assert.NoError(t, err)
+	assert.Equal(t, make([]byte, 16), buf)
 }
 
 func TestDecrypterClose(t *testing.T) {
@@ -1555,7 +1580,7 @@ func TestDecrypterClose(t *testing.T) {
 
 	// double close
 	err = fh.Close()
-	assert.Error(t, err, ErrorFileClosed.Error())
+	assert.EqualError(t, err, ErrorFileClosed.Error())
 	assert.Equal(t, 1, cd.closed)
 
 	// try again reading the file this time
@@ -1565,7 +1590,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := ioutil.ReadAll(fh)
+	out, err := io.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)
@@ -1582,8 +1607,6 @@ func TestPutGetBlock(t *testing.T) {
 	block := c.getBlock()
 	c.putBlock(block)
 	c.putBlock(block)
-
-	assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) })
 }
 
 func TestKey(t *testing.T) {

backend/crypt/crypt.go

@@ -28,6 +28,9 @@ func init() {
 		Description: "Encrypt/Decrypt a remote",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to encrypt/decrypt.\n\nNormally should contain a ':' and a path, e.g. \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -45,7 +48,7 @@ func init() {
 					Help:  "Very simple filename obfuscation.",
 				}, {
 					Value: "off",
-					Help:  "Don't encrypt the file names.\nAdds a \".bin\" extension only.",
+					Help:  "Don't encrypt the file names.\nAdds a \".bin\", or \"suffix\" extension only.",
 				},
 			},
 		}, {
@@ -76,7 +79,9 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 		}, {
 			Name:    "server_side_across_configs",
 			Default: false,
-			Help: `Allow server-side operations (e.g. copy) to work across different crypt configs.
+			Help: `Deprecated: use --server-side-across-configs instead.
+
+Allow server-side operations (e.g. copy) to work across different crypt configs.
 
 Normally this option is not what you want, but if you have two crypts
 pointing to the same backend you can use it.
@@ -116,13 +121,22 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
+		}, {
+			Name: "pass_bad_blocks",
+			Help: `If set this will pass bad blocks through as all 0.
+
+This should not be set in normal operation, it should only be set if
+trying to recover an encrypted file with errors and it is desired to
+recover as much of the file as possible.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
 
 This option could help with shortening the encrypted filename. The 
 suitable option would depend on the way your remote count the filename
-length and if it's case sensitve.`,
+length and if it's case sensitive.`,
 			Default: "base32",
 			Examples: []fs.OptionExample{
 				{
@@ -135,10 +149,18 @@ length and if it's case sensitve.`,
 				},
 				{
 					Value: "base32768",
-					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox)",
 				},
 			},
 			Advanced: true,
+		}, {
+			Name: "suffix",
+			Help: `If this is set it will override the default suffix of ".bin".
+
+Setting suffix to "none" will result in an empty suffix. This may be useful 
+when the path length is critical.`,
+			Default:  ".bin",
+			Advanced: true,
 		}},
 	})
 }
@@ -171,6 +193,8 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
+	cipher.setEncryptedSuffix(opt.Suffix)
+	cipher.setPassBadBlocks(opt.PassBadBlocks)
 	return cipher, nil
 }
 
@@ -232,7 +256,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         cipher.NameEncryptionMode() == NameEncryptionOff,
+		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
 		DuplicateFiles:          true,
 		ReadMimeType:            false, // MimeTypes not supported with crypt
 		WriteMimeType:           false,
@@ -241,6 +265,10 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		SetTier:                 true,
 		GetTier:                 true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -256,7 +284,9 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
+	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
+	Suffix                  string `config:"suffix"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -328,7 +358,7 @@ func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntr
 		case fs.Directory:
 			f.addDir(ctx, &newEntries, x)
 		default:
-			return nil, fmt.Errorf("Unknown object type %T", entry)
+			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
 	}
 	return newEntries, nil
@@ -390,6 +420,8 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	ci := fs.GetConfig(ctx)
+
 	if f.opt.NoDataEncryption {
 		o, err := put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
 		if err == nil && o != nil {
@@ -407,6 +439,9 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	// Find a hash the destination supports to compute a hash of
 	// the encrypted data
 	ht := f.Fs.Hashes().GetOne()
+	if ci.IgnoreChecksum {
+		ht = hash.None
+	}
 	var hasher *hash.MultiHasher
 	if ht != hash.None {
 		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
@@ -443,7 +478,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v crypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -501,9 +536,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -526,9 +561,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -597,7 +632,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.Fs.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -606,7 +641,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1041,10 +1076,11 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	// Get the underlying object if there is one
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
-		// Otherwise likely is an operations.OverrideRemote
+	} else if do, ok := o.ObjectInfo.(*fs.OverrideRemote); ok {
+		// Unwrap if it is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
+		// Otherwise don't unwrap any further
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash
@@ -1056,6 +1092,50 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	return "", nil
 }
 
+// GetTier returns storage tier or class of the Object
+func (o *ObjectInfo) GetTier() string {
+	do, ok := o.ObjectInfo.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *ObjectInfo) ID() string {
+	do, ok := o.ObjectInfo.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.ObjectInfo.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+//
+// This is deliberately unsupported so we don't leak mime type info by
+// default.
+func (o *ObjectInfo) MimeType(ctx context.Context) string {
+	return ""
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *ObjectInfo) UnWrap() fs.Object {
+	return fs.UnWrapObjectInfo(o.ObjectInfo)
+}
+
 // ID returns the ID of the Object if known, or "" if not
 func (o *Object) ID() string {
 	do, ok := o.Object.(fs.IDer)
@@ -1084,6 +1164,26 @@ func (o *Object) GetTier() string {
 	return do.GetTier()
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.Object.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+//
+// This is deliberately unsupported so we don't leak mime type info by
+// default.
+func (o *Object) MimeType(ctx context.Context) string {
+	return ""
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1106,10 +1206,6 @@ var (
 	_ fs.UserInfoer      = (*Fs)(nil)
 	_ fs.Disconnecter    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
+	_ fs.FullObjectInfo  = (*ObjectInfo)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/crypt/crypt_internal_test.go

@@ -17,41 +17,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type testWrapper struct {
-	fs.ObjectInfo
-}
-
-// UnWrap returns the Object that this Object is wrapping or nil if it
-// isn't wrapping anything
-func (o testWrapper) UnWrap() fs.Object {
-	if o, ok := o.ObjectInfo.(fs.Object); ok {
-		return o
-	}
-	return nil
-}
-
 // Create a temporary local fs to upload things from
 
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
 	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	}
-	return localFs, cleanup
+	})
+	return localFs
 }
 
 // Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
 	inBuf := bytes.NewBufferString(contents)
 	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
 	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
 	obj, err := f.Put(context.Background(), inBuf, upSrc)
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, obj.Remove(context.Background()))
-	}
-	return obj, cleanup
+	})
+	return obj
 }
 
 // Test the ObjectInfo
@@ -65,11 +52,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 		path = "_wrap"
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
-	obj, cleanupObj := uploadFile(t, localFs, path, contents)
-	defer cleanupObj()
+	obj := uploadFile(t, localFs, path, contents)
 
 	// encrypt the data
 	inBuf := bytes.NewBufferString(contents)
@@ -83,7 +68,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	var oi fs.ObjectInfo = obj
 	if wrap {
 		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = testWrapper{oi}
+		oi = fs.NewOverrideRemote(oi, "new_remote")
 	}
 
 	// wrap the object in a crypt for upload using the nonce we
@@ -91,7 +76,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	src := f.newObjectInfo(oi, nonce)
 
 	// Test ObjectInfo methods
-	assert.Equal(t, int64(outBuf.Len()), src.Size())
+	if !f.opt.NoDataEncryption {
+		assert.Equal(t, int64(outBuf.Len()), src.Size())
+	}
 	assert.Equal(t, f, src.Fs())
 	assert.NotEqual(t, path, src.Remote())
 
@@ -114,16 +101,13 @@ func testComputeHash(t *testing.T, f *Fs) {
 		t.Skipf("%v: does not support hashes", f.Fs)
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
 	// Upload a file to localFs as a test object
-	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
-	defer cleanupLocalObj()
+	localObj := uploadFile(t, localFs, path, contents)
 
 	// Upload the same data to the remote Fs also
-	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
-	defer cleanupRemoteObj()
+	remoteObj := uploadFile(t, f, path, contents)
 
 	// Calculate the expected Hash of the remote object
 	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)

backend/crypt/crypt_test.go

@@ -4,6 +4,7 @@ package crypt_test
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/rclone/rclone/backend/crypt"
@@ -46,6 +47,7 @@ func TestStandardBase32(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -67,6 +69,7 @@ func TestStandardBase64(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -88,6 +91,7 @@ func TestStandardBase32768(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -109,6 +113,7 @@ func TestOff(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -117,6 +122,9 @@ func TestObfuscate(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
+	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt3"
 	fstests.Run(t, &fstests.Opt{
@@ -131,6 +139,7 @@ func TestObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -139,6 +148,9 @@ func TestNoDataObfuscate(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
+	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt4"
 	fstests.Run(t, &fstests.Opt{
@@ -154,5 +166,6 @@ func TestNoDataObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }

backend/crypt/pkcs7/pkcs7.go

@@ -8,11 +8,11 @@ import "errors"
 
 // Errors Unpad can return
 var (
-	ErrorPaddingNotFound      = errors.New("Bad PKCS#7 padding - not padded")
-	ErrorPaddingNotAMultiple  = errors.New("Bad PKCS#7 padding - not a multiple of blocksize")
-	ErrorPaddingTooLong       = errors.New("Bad PKCS#7 padding - too long")
-	ErrorPaddingTooShort      = errors.New("Bad PKCS#7 padding - too short")
-	ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same")
+	ErrorPaddingNotFound      = errors.New("bad PKCS#7 padding - not padded")
+	ErrorPaddingNotAMultiple  = errors.New("bad PKCS#7 padding - not a multiple of blocksize")
+	ErrorPaddingTooLong       = errors.New("bad PKCS#7 padding - too long")
+	ErrorPaddingTooShort      = errors.New("bad PKCS#7 padding - too short")
+	ErrorPaddingNotAllTheSame = errors.New("bad PKCS#7 padding - not all the same")
 )
 
 // Pad buf using PKCS#7 to a multiple of n.

backend/drive/drive.go

@@ -14,9 +14,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"path"
 	"sort"
 	"strconv"
@@ -50,6 +50,7 @@ import (
 	drive_v2 "google.golang.org/api/drive/v2"
 	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
 )
 
 // Constants
@@ -70,7 +71,7 @@ const (
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	minChunkSize     = fs.SizeSuffix(googleapi.MinUploadChunkSize)
 	defaultChunkSize = 8 * fs.Mebi
-	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
+	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
 	defaultXDGIcon   = "text-html"
@@ -201,7 +202,7 @@ func init() {
 					m.Set("root_folder_id", "appDataFolder")
 				}
 
-				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
+				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
 					return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
 						OAuth2Config: driveConfig,
 					})
@@ -276,19 +277,23 @@ Leave blank normally.
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 `,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
-			Name:     "service_account_credentials",
-			Help:     "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
-			Hide:     fs.OptionHideConfigurator,
-			Advanced: true,
+			Name:      "service_account_credentials",
+			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Hide:      fs.OptionHideConfigurator,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "team_drive",
-			Help:     "ID of the Shared Drive (Team Drive).",
-			Hide:     fs.OptionHideConfigurator,
-			Advanced: true,
+			Name:      "team_drive",
+			Help:      "ID of the Shared Drive (Team Drive).",
+			Hide:      fs.OptionHideConfigurator,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "auth_owner_only",
 			Default:  false,
@@ -414,10 +419,11 @@ date is used.`,
 			Help:     "Size of listing chunk 100-1000, 0 to disable.",
 			Advanced: true,
 		}, {
-			Name:     "impersonate",
-			Default:  "",
-			Help:     `Impersonate this user when using a service account.`,
-			Advanced: true,
+			Name:      "impersonate",
+			Default:   "",
+			Help:      `Impersonate this user when using a service account.`,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:    "alternate_export",
 			Default: false,
@@ -449,7 +455,11 @@ If downloading a file returns the error "This file has been identified
 as malware or spam and cannot be downloaded" with the error code
 "cannotDownloadAbusiveFile" then supply this flag to rclone to
 indicate you acknowledge the risks of downloading the file and rclone
-will download it anyway.`,
+will download it anyway.
+
+Note that if you are using service account it will need Manager
+permission (not Content Manager) to for this flag to work. If the SA
+does not have the right permission, Google will just ignore the flag.`,
 			Advanced: true,
 		}, {
 			Name:     "keep_revision_forever",
@@ -493,7 +503,9 @@ need to use --ignore size also.`,
 		}, {
 			Name:    "server_side_across_configs",
 			Default: false,
-			Help: `Allow server-side operations (e.g. copy) to work across different drive configs.
+			Help: `Deprecated: use --server-side-across-configs instead.
+
+Allow server-side operations (e.g. copy) to work across different drive configs.
 
 This can be useful if you wish to do a server-side copy between two
 different Google drives.  Note that this isn't enabled by default
@@ -564,6 +576,28 @@ If this is set then rclone will not show any dangling shortcuts in listings.
 `,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name: "resource_key",
+			Help: `Resource key for accessing a link-shared file.
+
+If you need to access files shared with a link like this
+
+    https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing
+
+Then you will need to use the first part "XXX" as the "root_folder_id"
+and the second part "YYY" as the "resource_key" otherwise you will get
+404 not found errors when trying to access the directory.
+
+See: https://developers.google.com/drive/api/guides/resource-keys
+
+This resource key requirement only applies to a subset of old files.
+
+Note also that opening the folder once in the web interface (with the
+user you've authenticated rclone with) seems to be enough so that the
+resource key is no needed.
+`,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -571,6 +605,18 @@ If this is set then rclone will not show any dangling shortcuts in listings.
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// Don't encode / as it's a valid name character in drive.
 			Default: encoder.EncodeInvalidUtf8,
+		}, {
+			Name:     "env_auth",
+			Help:     "Get IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 
@@ -625,7 +671,9 @@ type Options struct {
 	StopOnDownloadLimit       bool                 `config:"stop_on_download_limit"`
 	SkipShortcuts             bool                 `config:"skip_shortcuts"`
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
+	ResourceKey               string               `config:"resource_key"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote drive server
@@ -650,6 +698,7 @@ type Fs struct {
 	grouping         int32               // number of IDs to search at once in ListR - read with atomic
 	listRmu          *sync.Mutex         // protects listRempties
 	listRempties     map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
+	dirResourceKeys  *sync.Map           // map directory ID to resource key
 }
 
 type baseObject struct {
@@ -660,6 +709,7 @@ type baseObject struct {
 	mimeType     string   // The object MIME type
 	bytes        int64    // size of the object
 	parents      []string // IDs of the parent directories
+	resourceKey  *string  // resourceKey is needed for link shared objects
 }
 type documentObject struct {
 	baseObject
@@ -731,6 +781,9 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
+			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
+				fs.Errorf(f, "Received upload limit error: %v", err)
+				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
 				fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
 				return false, fserrors.FatalError(err)
@@ -800,6 +853,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
 	// If we need to list file inside those shared folders, we must search it without sharedWithMe
 	parentsQuery := bytes.NewBufferString("(")
+	var resourceKeys []string
 	for _, dirID := range dirIDs {
 		if dirID == "" {
 			continue
@@ -820,7 +874,12 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		} else {
 			_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
 		}
+		resourceKey, hasResourceKey := f.dirResourceKeys.Load(dirID)
+		if hasResourceKey {
+			resourceKeys = append(resourceKeys, fmt.Sprintf("%s/%s", dirID, resourceKey))
+		}
 	}
+	resourceKeysHeader := strings.Join(resourceKeys, ",")
 	if parentsQuery.Len() > 1 {
 		_ = parentsQuery.WriteByte(')')
 		query = append(query, parentsQuery.String())
@@ -829,8 +888,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	if title != "" {
 		searchTitle := f.opt.Enc.FromStandardName(title)
 		// Escaping the backslash isn't documented but seems to work
-		searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
-		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
+		searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
+		searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
 
 		var titleQuery bytes.Buffer
 		_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
@@ -884,7 +943,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	}
 	list.SupportsAllDrives(true)
 	list.IncludeItemsFromAllDrives(true)
-	if f.isTeamDrive {
+	if f.isTeamDrive && !f.opt.SharedWithMe {
 		list.DriveId(f.opt.TeamDriveID)
 		list.Corpora("drive")
 	}
@@ -892,6 +951,10 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	if f.rootFolderID == "appDataFolder" {
 		list.Spaces("appDataFolder")
 	}
+	// Add resource Keys if necessary
+	if resourceKeysHeader != "" {
+		list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader)
+	}
 
 	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields)
 
@@ -1068,7 +1131,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -1079,6 +1142,12 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 		if err != nil {
 			return nil, fmt.Errorf("failed to create oauth client from service account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		scopes := driveScopes(opt.Scope)
+		oAuthClient, err = google.DefaultClient(ctx, scopes...)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create client from environment: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
 		if err != nil {
@@ -1151,15 +1220,16 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:         name,
-		root:         root,
-		opt:          *opt,
-		ci:           ci,
-		pacer:        fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
-		m:            m,
-		grouping:     listRGrouping,
-		listRmu:      new(sync.Mutex),
-		listRempties: make(map[string]struct{}),
+		name:            name,
+		root:            root,
+		opt:             *opt,
+		ci:              ci,
+		pacer:           fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
+		m:               m,
+		grouping:        listRGrouping,
+		listRmu:         new(sync.Mutex),
+		listRempties:    make(map[string]struct{}),
+		dirResourceKeys: new(sync.Map),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.fileFields = f.getFileFields()
@@ -1169,17 +1239,18 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
-	f.svc, err = drive.New(f.client)
+	f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create Drive client: %w", err)
 	}
 
 	if f.opt.V2DownloadMinSize >= 0 {
-		f.v2Svc, err = drive_v2.New(f.client)
+		f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
 		if err != nil {
 			return nil, fmt.Errorf("couldn't create Drive v2 client: %w", err)
 		}
@@ -1221,6 +1292,11 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 
 	f.dirCache = dircache.New(f.root, f.rootFolderID, f)
 
+	// If resource key is set then cache it for the root folder id
+	if f.opt.ResourceKey != "" {
+		f.dirResourceKeys.Store(f.rootFolderID, f.opt.ResourceKey)
+	}
+
 	// Parse extensions
 	if f.opt.Extensions != "" {
 		if f.opt.ExportExtensions != defaultExportExtensions {
@@ -1319,12 +1395,16 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
 			}
 		}
 	}
-	return &Object{
+	o := &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
 		md5sum:     strings.ToLower(info.Md5Checksum),
 		v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
 	}
+	if info.ResourceKey != "" {
+		o.resourceKey = &info.ResourceKey
+	}
+	return o
 }
 
 // newDocumentObject creates an fs.Object for a google docs drive.File
@@ -1439,6 +1519,9 @@ func (f *Fs) newObjectWithExportInfo(
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	if strings.HasSuffix(remote, "/") {
+		return nil, fs.ErrorIsDir
+	}
 	info, extension, exportName, exportMimeType, isDocument, err := f.getRemoteInfoWithExport(ctx, remote)
 	if err != nil {
 		return nil, err
@@ -2014,7 +2097,7 @@ func splitID(compositeID string) (actualID, shortcutID string) {
 
 // isShortcutID returns true if compositeID refers to a shortcut
 func isShortcutID(compositeID string) bool {
-	return strings.IndexRune(compositeID, shortcutSeparator) >= 0
+	return strings.ContainsRune(compositeID, shortcutSeparator)
 }
 
 // actualID returns an actual ID from a composite ID
@@ -2085,6 +2168,10 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File
 	case item.MimeType == driveFolderType:
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, item.Id)
+		// cache the resource key for later lookups
+		if item.ResourceKey != "" {
+			f.dirResourceKeys.Store(item.Id, item.ResourceKey)
+		}
 		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
 		d := fs.NewDir(remote, when).SetID(item.Id)
 		if len(item.Parents) > 0 {
@@ -2126,7 +2213,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -2168,10 +2255,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 			exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
 			if exportExt == "" {
-				return nil, fmt.Errorf("No export format found for %q", importMimeType)
+				return nil, fmt.Errorf("no export format found for %q", importMimeType)
 			}
 			if exportExt != srcExt && !f.opt.AllowImportNameChange {
-				return nil, fmt.Errorf("Can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt)
+				return nil, fmt.Errorf("can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt)
 			}
 		}
 	}
@@ -2360,9 +2447,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2429,11 +2516,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Copy(id, createInfo).
+		copy := f.svc.Files.Copy(id, createInfo).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			KeepRevisionForever(f.opt.KeepRevisionForever).
-			Context(ctx).Do()
+			KeepRevisionForever(f.opt.KeepRevisionForever)
+		srcObj.addResourceKey(copy.Header())
+		info, err = copy.Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -2475,7 +2563,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // result of List()
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	if f.opt.TrashedOnly {
-		return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files")
+		return errors.New("can't purge with --drive-trashed-only, use delete if you want to selectively delete files")
 	}
 	return f.purgeCheck(ctx, dir, false)
 }
@@ -2594,9 +2682,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2821,6 +2909,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			if f.rootFolderID == "appDataFolder" {
 				changesCall.Spaces("appDataFolder")
 			}
+			changesCall.RestrictToMyDrive(!f.opt.SharedWithMe)
 			changeList, err = changesCall.Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
@@ -2941,12 +3030,12 @@ func (f *Fs) changeServiceAccountFile(ctx context.Context, file string) (err err
 		return fmt.Errorf("drive: failed when making oauth client: %w", err)
 	}
 	f.client = oAuthClient
-	f.svc, err = drive.New(f.client)
+	f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
 	if err != nil {
 		return fmt.Errorf("couldn't create Drive client: %w", err)
 	}
 	if f.opt.V2DownloadMinSize >= 0 {
-		f.v2Svc, err = drive_v2.New(f.client)
+		f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
 		if err != nil {
 			return fmt.Errorf("couldn't create Drive v2 client: %w", err)
 		}
@@ -3235,7 +3324,7 @@ This will return a JSON list of objects like this
 
 With the -o config parameter it will output the list in a format
 suitable for adding to a config file to make aliases for all the
-drives found.
+drives found and a combined drive.
 
     [My Drive]
     type = alias
@@ -3245,10 +3334,15 @@ drives found.
     type = alias
     remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
 
-Adding this to the rclone config file will cause those team drives to
-be accessible with the aliases shown. This may require manual editing
-of the names.
+    [AllDrives]
+    type = combine
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
+Adding this to the rclone config file will cause those team drives to
+be accessible with the aliases shown. Any illegal characters will be
+substituted with "_" and duplicate names will have numbers suffixed.
+It will also add a remote called AllDrives which shows all the shared
+drives combined into one directory tree.
 `,
 }, {
 	Name:  "untrash",
@@ -3262,9 +3356,9 @@ This takes an optional directory to trash which make this easier to
 use via the API.
 
     rclone backend untrash drive:directory
-    rclone backend -i untrash drive:directory subdir
+    rclone backend --interactive untrash drive:directory subdir
 
-Use the -i flag to see what would be restored before restoring it.
+Use the --interactive/-i or --dry-run flag to see what would be restored before restoring it.
 
 Result:
 
@@ -3294,8 +3388,14 @@ component will be used as the file name.
 If the destination is a drive backend then server-side copying will be
 attempted if possible.
 
-Use the -i flag to see what would be copied before copying.
+Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
 `,
+}, {
+	Name:  "exportformats",
+	Short: "Dump the export formats for debug purposes",
+}, {
+	Name:  "importformats",
+	Short: "Dump the import formats for debug purposes",
 }}
 
 // Command the backend to run a named command
@@ -3315,7 +3415,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			out["service_account_file"] = f.opt.ServiceAccountFile
 		}
 		if _, ok := opt["chunk_size"]; ok {
-			out["chunk_size"] = fmt.Sprintf("%s", f.opt.ChunkSize)
+			out["chunk_size"] = f.opt.ChunkSize.String()
 		}
 		return out, nil
 	case "set":
@@ -3332,11 +3432,11 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		}
 		if chunkSize, ok := opt["chunk_size"]; ok {
 			chunkSizeMap := make(map[string]string)
-			chunkSizeMap["previous"] = fmt.Sprintf("%s", f.opt.ChunkSize)
+			chunkSizeMap["previous"] = f.opt.ChunkSize.String()
 			if err = f.changeChunkSize(chunkSize); err != nil {
 				return out, err
 			}
-			chunkSizeString := fmt.Sprintf("%s", f.opt.ChunkSize)
+			chunkSizeString := f.opt.ChunkSize.String()
 			f.m.Set("chunk_size", chunkSizeString)
 			chunkSizeMap["current"] = chunkSizeString
 			out["chunk_size"] = chunkSizeMap
@@ -3366,12 +3466,27 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		}
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
-			for _, drive := range drives {
+			upstreams := []string{}
+			names := make(map[string]struct{}, len(drives))
+			for i, drive := range drives {
+				name := fspath.MakeConfigName(drive.Name)
+				for {
+					if _, found := names[name]; !found {
+						break
+					}
+					name += fmt.Sprintf("-%d", i)
+				}
+				names[name] = struct{}{}
 				lines = append(lines, "")
-				lines = append(lines, fmt.Sprintf("[%s]", drive.Name))
-				lines = append(lines, fmt.Sprintf("type = alias"))
+				lines = append(lines, fmt.Sprintf("[%s]", name))
+				lines = append(lines, "type = alias")
 				lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id))
+				upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name))
 			}
+			lines = append(lines, "")
+			lines = append(lines, "[AllDrives]")
+			lines = append(lines, "type = combine")
+			lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " ")))
 			return lines, nil
 		}
 		return drives, nil
@@ -3394,6 +3509,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			}
 		}
 		return nil, nil
+	case "exportformats":
+		return f.exportFormats(ctx), nil
+	case "importformats":
+		return f.importFormats(ctx), nil
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -3443,12 +3562,6 @@ func (o *baseObject) Size() int64 {
 	return o.bytes
 }
 
-// getRemoteInfo returns a drive.File for the remote
-func (f *Fs) getRemoteInfo(ctx context.Context, remote string) (info *drive.File, err error) {
-	info, _, _, _, _, err = f.getRemoteInfoWithExport(ctx, remote)
-	return
-}
-
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
@@ -3489,7 +3602,6 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *baseObject) ModTime(ctx context.Context) time.Time {
@@ -3530,6 +3642,14 @@ func (o *baseObject) Storable() bool {
 	return true
 }
 
+// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
+// object if required.
+func (o *baseObject) addResourceKey(header http.Header) {
+	if o.resourceKey != nil {
+		header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
+	}
+}
+
 // httpResponse gets an http.Response object for the object
 // using the url and method passed in
 func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
@@ -3545,6 +3665,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 		// Don't supply range requests for 0 length objects as they always fail
 		delete(req.Header, "Range")
 	}
+	o.addResourceKey(req.Header)
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {
@@ -3624,7 +3745,7 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
 				url += "acknowledgeAbuse=true"
 				_, res, err = o.httpResponse(ctx, url, "GET", options)
 			} else {
-				err = fmt.Errorf("Use the --drive-acknowledge-abuse flag to download this file: %w", err)
+				err = fmt.Errorf("use the --drive-acknowledge-abuse flag to download this file: %w", err)
 			}
 		}
 		if err != nil {
@@ -3711,7 +3832,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(data)), nil
+	return io.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
@@ -3737,7 +3858,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
@@ -3770,7 +3891,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
+	newO, err := o.fs.newObjectWithInfo(ctx, o.remote, info)
 	if err != nil {
 		return err
 	}

backend/drive/drive_internal_test.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -19,6 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/sync"
@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
 )
 
 func TestDriveScopes(t *testing.T) {
@@ -76,7 +77,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -190,6 +191,69 @@ func TestExtensionsForImportFormats(t *testing.T) {
 	}
 }
 
+func (f *Fs) InternalTestShouldRetry(t *testing.T) {
+	ctx := context.Background()
+	gatewayTimeout := googleapi.Error{
+		Code: 503,
+	}
+	timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout)
+	assert.True(t, timeoutRetry)
+	assert.Equal(t, &gatewayTimeout, timeoutError)
+	generic403 := googleapi.Error{
+		Code: 403,
+	}
+	rLEItem := googleapi.ErrorItem{
+		Reason:  "rateLimitExceeded",
+		Message: "User rate limit exceeded.",
+	}
+	generic403.Errors = append(generic403.Errors, rLEItem)
+	oldStopUpload := f.opt.StopOnUploadLimit
+	oldStopDownload := f.opt.StopOnDownloadLimit
+	f.opt.StopOnUploadLimit = true
+	f.opt.StopOnDownloadLimit = true
+	defer func() {
+		f.opt.StopOnUploadLimit = oldStopUpload
+		f.opt.StopOnDownloadLimit = oldStopDownload
+	}()
+	expectedRLError := fserrors.FatalError(&generic403)
+	rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403)
+	assert.False(t, rateLimitRetry)
+	assert.Equal(t, rateLimitErr, expectedRLError)
+	dQEItem := googleapi.ErrorItem{
+		Reason: "downloadQuotaExceeded",
+	}
+	generic403.Errors[0] = dQEItem
+	expectedDQError := fserrors.FatalError(&generic403)
+	downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, downloadQuotaRetry)
+	assert.Equal(t, downloadQuotaError, expectedDQError)
+	tDFLEItem := googleapi.ErrorItem{
+		Reason: "teamDriveFileLimitExceeded",
+	}
+	generic403.Errors[0] = tDFLEItem
+	expectedTDFLError := fserrors.FatalError(&generic403)
+	teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, teamDriveFileLimitRetry)
+	assert.Equal(t, teamDriveFileLimitError, expectedTDFLError)
+	qEItem := googleapi.ErrorItem{
+		Reason: "quotaExceeded",
+	}
+	generic403.Errors[0] = qEItem
+	expectedQuotaError := fserrors.FatalError(&generic403)
+	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, quotaExceededRetry)
+	assert.Equal(t, quotaExceededError, expectedQuotaError)
+
+	sqEItem := googleapi.ErrorItem{
+		Reason: "storageQuotaExceeded",
+	}
+	generic403.Errors[0] = sqEItem
+	expectedStorageQuotaError := fserrors.FatalError(&generic403)
+	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, storageQuotaExceededRetry)
+	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
+}
+
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	oldAllow := f.opt.AllowImportNameChange
 	f.opt.AllowImportNameChange = true
@@ -378,9 +442,9 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) {
 	// Make some objects, one in a subdir
 	contents := random.String(100)
 	file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
-	_, obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
+	obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
 	file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
-	_, _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
+	_ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
 
 	// Check objects
 	checkObjects := func() {
@@ -462,6 +526,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
@@ -496,7 +563,7 @@ func (f *Fs) InternalTestAgeQuery(t *testing.T) {
 	require.NoError(t, err)
 
 	file1 := fstest.Item{ModTime: time.Now(), Path: "agequery.txt"}
-	_, _ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
+	_ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
 
 	// validate sync/copy
 	const timeQuery = "(modifiedTime >= '"
@@ -545,6 +612,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
+	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/dropbox/batcher.go

@@ -13,7 +13,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -118,12 +117,12 @@ func (b *batcher) Batching() bool {
 }
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
+func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
 	var arg = &files.UploadSessionFinishBatchArg{
 		Entries: items,
 	}
 	err = b.f.pacer.Call(func() (bool, error) {
-		batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
+		complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
 		// If error is insufficient space then don't retry
 		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -137,50 +136,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
 	if err != nil {
 		return nil, fmt.Errorf("batch commit failed: %w", err)
 	}
-	return batchStatus, nil
-}
-
-// finishBatchJobStatus waits for the batch to complete returning completed entries
-func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *files.UploadSessionFinishBatchLaunch) (complete *files.UploadSessionFinishBatchResult, err error) {
-	if launchBatchStatus.AsyncJobId == "" {
-		return nil, errors.New("wait for batch completion: empty job ID")
-	}
-	var batchStatus *files.UploadSessionFinishBatchJobStatus
-	sleepTime := 100 * time.Millisecond
-	const maxSleepTime = 1 * time.Second
-	startTime := time.Now()
-	try := 1
-	for {
-		remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
-		if remaining < 0 {
-			break
-		}
-		err = b.f.pacer.Call(func() (bool, error) {
-			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
-				AsyncJobId: launchBatchStatus.AsyncJobId,
-			})
-			return shouldRetry(ctx, err)
-		})
-		if err != nil {
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
-		} else {
-			if batchStatus.Tag == "complete" {
-				fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
-				return batchStatus.Complete, nil
-			}
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
-		}
-		time.Sleep(sleepTime)
-		sleepTime *= 2
-		if sleepTime > maxSleepTime {
-			sleepTime = maxSleepTime
-		}
-		try++
-	}
-	if err == nil {
-		err = errors.New("batch didn't complete")
-	}
-	return nil, fmt.Errorf("wait for batch failed after %d tries in %v: %w", try, time.Since(startTime), err)
+	return complete, nil
 }
 
 // commit a batch
@@ -188,7 +144,7 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
 	// If commit fails then signal clients if sync
 	var signalled = b.async
 	defer func() {
-		if err != nil && signalled {
+		if err != nil && !signalled {
 			// Signal to clients that there was an error
 			for _, result := range results {
 				result <- batcherResponse{err: err}
@@ -199,26 +155,11 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
 	fs.Debugf(b.f, "Committing %s", desc)
 
 	// finalise the batch getting either a result or a job id to poll
-	batchStatus, err := b.finishBatch(ctx, items)
+	complete, err := b.finishBatch(ctx, items)
 	if err != nil {
 		return err
 	}
 
-	// check whether batch is complete
-	var complete *files.UploadSessionFinishBatchResult
-	switch batchStatus.Tag {
-	case "async_job_id":
-		// wait for batch to complete
-		complete, err = b.finishBatchJobStatus(ctx, batchStatus)
-		if err != nil {
-			return err
-		}
-	case "complete":
-		complete = batchStatus.Complete
-	default:
-		return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
-	}
-
 	// Check we got the right number of entries
 	entries := complete.Entries
 	if len(entries) != len(results) {
@@ -319,9 +260,12 @@ outer:
 //
 // Can be called from atexit handler
 func (b *batcher) Shutdown() {
+	if !b.Batching() {
+		return
+	}
 	b.shutOnce.Do(func() {
 		atexit.Unregister(b.atexit)
-		fs.Infof(b.f, "Commiting uploads - please wait...")
+		fs.Infof(b.f, "Committing uploads - please wait...")
 		// show that batcher is shutting down
 		close(b.closed)
 		// quit the commitLoop by sending a quitRequest message

backend/dropbox/dropbox.go

@@ -58,7 +58,7 @@ import (
 const (
 	rcloneClientID              = "5jcck7diasz0rqy"
 	rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
-	minSleep                    = 10 * time.Millisecond
+	defaultMinSleep             = fs.Duration(10 * time.Millisecond)
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	// Upload chunk size - setting too small makes uploads slow.
@@ -182,8 +182,9 @@ client_secret) to use this option as currently rclone's default set of
 permissions doesn't include "members.read". This can be added once
 v1.55 or later is in use everywhere.
 `,
-			Default:  "",
-			Advanced: true,
+			Default:   "",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "shared_files",
 			Help: `Instructs rclone to work on individual shared files.
@@ -260,17 +261,22 @@ uploaded.
 The default for this is 0 which means rclone will choose a sensible
 default based on the batch_mode in use.
 
-- batch_mode: async - default batch_timeout is 500ms
-- batch_mode: sync - default batch_timeout is 10s
+- batch_mode: async - default batch_timeout is 10s
+- batch_mode: sync - default batch_timeout is 500ms
 - batch_mode: off - not in use
 `,
 			Default:  fs.Duration(0),
 			Advanced: true,
 		}, {
 			Name:     "batch_commit_timeout",
-			Help:     `Max time to wait for a batch to finish comitting`,
+			Help:     `Max time to wait for a batch to finish committing`,
 			Default:  fs.Duration(10 * time.Minute),
 			Advanced: true,
+		}, {
+			Name:     "pacer_min_sleep",
+			Default:  defaultMinSleep,
+			Help:     "Minimum time to sleep between API calls.",
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -299,6 +305,7 @@ type Options struct {
 	BatchTimeout       fs.Duration          `config:"batch_timeout"`
 	BatchCommitTimeout fs.Duration          `config:"batch_commit_timeout"`
 	AsyncBatch         bool                 `config:"async_batch"`
+	PacerMinSleep      fs.Duration          `config:"pacer_min_sleep"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -442,7 +449,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:  name,
 		opt:   *opt,
 		ci:    ci,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.batcher, err = newBatcher(ctx, f, f.opt.BatchMode, f.opt.BatchSize, time.Duration(f.opt.BatchTimeout))
 	if err != nil {
@@ -472,10 +479,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		args := team.NewMembersGetInfoArgs(members)
 
 		memberIds, err := f.team.MembersGetInfo(args)
-
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
+		if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil {
+			return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate)
+		}
 
 		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
@@ -534,7 +543,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			default:
 				return nil, err
 			}
-			// if the moint failed we have to abort here
+			// if the mount failed we have to abort here
 		}
 		// if the mount succeeded it's now a normal folder in the users root namespace
 		// we disable shared folder mode and proceed normally
@@ -717,7 +726,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
 		}
 		for _, entry := range res.Entries {
 			leaf := f.opt.Enc.ToStandardName(entry.Name)
-			d := fs.NewDir(leaf, time.Now()).SetID(entry.SharedFolderId)
+			d := fs.NewDir(leaf, time.Time{}).SetID(entry.SharedFolderId)
 			entries = append(entries, d)
 			if err != nil {
 				return nil, err
@@ -904,7 +913,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
-				d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
+				d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
 				entries = append(entries, d)
 			} else if fileInfo != nil {
 				o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
@@ -923,7 +932,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1042,9 +1051,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1103,9 +1112,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1197,7 +1206,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 			return
 		}
 		if len(listRes.Links) == 0 {
-			err = errors.New("Dropbox says the sharing link already exists, but list came back empty")
+			err = errors.New("sharing link already exists, but list came back empty")
 			return
 		}
 		linkRes = listRes.Links[0]
@@ -1209,7 +1218,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		case *sharing.FolderLinkMetadata:
 			link = res.Url
 		default:
-			err = fmt.Errorf("Don't know how to extract link, response has unknown format: %T", res)
+			err = fmt.Errorf("don't know how to extract link, response has unknown format: %T", res)
 		}
 	}
 	return
@@ -1269,7 +1278,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("about failed: %w", err)
+		return nil, err
 	}
 	var total uint64
 	if q.Allocation != nil {
@@ -1370,10 +1379,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 
 	if timeout < 30 {
 		timeout = 30
+		fs.Debugf(f, "Increasing poll interval to minimum 30s")
 	}
 
 	if timeout > 480 {
 		timeout = 480
+		fs.Debugf(f, "Decreasing poll interval to maximum 480s")
 	}
 
 	err = f.pacer.Call(func() (bool, error) {
@@ -1431,7 +1442,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			}
 
 			if entryPath != "" {
-				notifyFunc(entryPath, entryType)
+				notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
 			}
 		}
 		if !changeList.HasMore {
@@ -1665,7 +1676,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 						correctOffset := uErr.EndpointError.IncorrectOffset.CorrectOffset
 						delta := int64(correctOffset) - int64(cursor.Offset)
 						skip += delta
-						what := fmt.Sprintf("incorrect offset error receved: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
+						what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
 						if skip < 0 {
 							return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
 						} else if skip == chunkSize {
@@ -1693,6 +1704,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 		if size > 0 {
 			// if size is known, check if next chunk is final
 			appendArg.Close = uint64(size)-in.BytesRead() <= uint64(chunkSize)
+			if in.BytesRead() > uint64(size) {
+				return nil, fmt.Errorf("expected %d bytes in input, but have read %d so far", size, in.BytesRead())
+			}
 		} else {
 			// if size is unknown, upload as long as we can read full chunks from the reader
 			appendArg.Close = in.BytesRead()-cursor.Offset < uint64(chunkSize)
@@ -1756,7 +1770,7 @@ func checkPathLength(name string) (err error) {
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/fichier/api.go

@@ -28,25 +28,44 @@ var retryErrorCodes = []int{
 	509, // Bandwidth Limit Exceeded
 }
 
+var errorRegex = regexp.MustCompile(`#\d{1,3}`)
+
+func parseFichierError(err error) int {
+	matches := errorRegex.FindStringSubmatch(err.Error())
+	if len(matches) == 0 {
+		return 0
+	}
+	code, err := strconv.Atoi(matches[0])
+	if err != nil {
+		fs.Debugf(nil, "failed parsing fichier error: %v", err)
+		return 0
+	}
+	return code
+}
+
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	// Detect this error which the integration tests provoke
-	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
+	// 1Fichier uses HTTP error code 403 (Forbidden) for all kinds of errors with
+	// responses looking like this: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
 	//
-	// https://1fichier.com/api.html
-	//
-	// file/ls.cgi is limited :
-	//
-	// Warning (can be changed in case of abuses) :
-	// List all files of the account is limited to 1 request per hour.
-	// List folders is limited to 5 000 results and 1 request per folder per 30s.
-	if err != nil && strings.Contains(err.Error(), "Flood detected") {
-		fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
-		time.Sleep(30 * time.Second)
+	// We attempt to parse the actual 1Fichier error code from this body and handle it accordingly
+	// Most importantly #374 (Flood detected: IP locked) which the integration tests provoke
+	// The list below is far from complete and should be expanded if we see any more error codes.
+	if err != nil {
+		switch parseFichierError(err) {
+		case 93:
+			return false, err // No such user
+		case 186:
+			return false, err // IP blocked?
+		case 374:
+			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
+			time.Sleep(30 * time.Second)
+		default:
+		}
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -99,6 +118,9 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
 		Single: 1,
 		Pass:   f.opt.FilePassword,
 	}
+	if f.opt.CDN {
+		request.CDN = 1
+	}
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/download/get_token.cgi",
@@ -386,6 +408,32 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
 	return response, nil
 }
 
+func (f *Fs) moveDir(ctx context.Context, folderID int, newLeaf string, destinationFolderID int) (response *MoveDirResponse, err error) {
+	request := &MoveDirRequest{
+		FolderID:            folderID,
+		DestinationFolderID: destinationFolderID,
+		Rename:              newLeaf,
+		// DestinationUser:     destinationUser,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/mv.cgi",
+	}
+
+	response = &MoveDirResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
+		return shouldRetry(ctx, resp, err)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("couldn't move dir: %w", err)
+	}
+
+	return response, nil
+}
+
 func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
 	request := &CopyFileRequest{
 		URLs:     []string{url},
@@ -454,7 +502,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("didnt got an upload node: %w", err)
+		return nil, fmt.Errorf("didn't get an upload node: %w", err)
 	}
 
 	// fs.Debugf(f, "Got Upload node")
@@ -468,7 +516,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 	fileName = f.opt.Enc.FromStandardName(fileName)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
+		return nil, errors.New("invalid UploadID")
 	}
 
 	opts := rest.Opts{
@@ -510,7 +558,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
 	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
+		return nil, errors.New("invalid UploadID")
 	}
 
 	opts := rest.Opts{

backend/fichier/fichier.go

@@ -1,3 +1,4 @@
+// Package fichier provides an interface to the 1Fichier storage system.
 package fichier
 
 import (
@@ -37,8 +38,9 @@ func init() {
 		Description: "1Fichier",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Help: "Your API Key, get it from https://1fichier.com/console/params.pl.",
-			Name: "api_key",
+			Help:      "Your API Key, get it from https://1fichier.com/console/params.pl.",
+			Name:      "api_key",
+			Sensitive: true,
 		}, {
 			Help:     "If you want to download a shared folder, add this parameter.",
 			Name:     "shared_folder",
@@ -53,6 +55,11 @@ func init() {
 			Name:       "folder_password",
 			Advanced:   true,
 			IsPassword: true,
+		}, {
+			Help:     "Set if you wish to use CDN download links.",
+			Name:     "cdn",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -88,6 +95,7 @@ type Options struct {
 	SharedFolder   string               `config:"shared_folder"`
 	FilePassword   string               `config:"file_password"`
 	FolderPassword string               `config:"folder_password"`
+	CDN            bool                 `config:"cdn"`
 	Enc            encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -294,7 +302,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 			path, ok := f.dirCache.GetInv(directoryID)
 
 			if !ok {
-				return nil, errors.New("Cannot find dir in dircache")
+				return nil, errors.New("cannot find dir in dircache")
 			}
 
 			return f.newObjectFromFile(ctx, path, file), nil
@@ -332,7 +340,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
 	if size > int64(300e9) {
-		return nil, errors.New("File too big, cant upload")
+		return nil, errors.New("File too big, can't upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles
 	}
@@ -480,6 +488,51 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove.
+//
+// If destination exists then return fs.ErrorDirExists.
+//
+// This is complicated by the fact that we can't use moveDir to move
+// to a different directory AND rename at the same time as it can
+// overwrite files in the source directory.
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+	if err != nil {
+		return err
+	}
+	srcIDnumeric, err := strconv.Atoi(srcID)
+	if err != nil {
+		return err
+	}
+	dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID)
+	if err != nil {
+		return err
+	}
+
+	var resp *MoveDirResponse
+	resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric)
+	if err != nil {
+		return fmt.Errorf("couldn't rename leaf: %w", err)
+	}
+	if resp.Status != "OK" {
+		return fmt.Errorf("couldn't rename leaf: %s", resp.Message)
+	}
+
+	srcFs.dirCache.FlushDir(srcRemote)
+	return nil
+}
+
 // Copy src to this remote using server side move operations.
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
@@ -553,6 +606,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 var (
 	_ fs.Fs              = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)

backend/fichier/structs.go

@@ -20,6 +20,7 @@ type DownloadRequest struct {
 	URL    string `json:"url"`
 	Single int    `json:"single"`
 	Pass   string `json:"pass,omitempty"`
+	CDN    int    `json:"cdn,omitempty"`
 }
 
 // RemoveFolderRequest is the request structure of the corresponding request
@@ -69,6 +70,22 @@ type MoveFileResponse struct {
 	URLs    []string `json:"urls"`
 }
 
+// MoveDirRequest is the request structure of the corresponding request
+type MoveDirRequest struct {
+	FolderID            int    `json:"folder_id"`
+	DestinationFolderID int    `json:"destination_folder_id,omitempty"`
+	DestinationUser     string `json:"destination_user"`
+	Rename              string `json:"rename,omitempty"`
+}
+
+// MoveDirResponse is the response structure of the corresponding request
+type MoveDirResponse struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	OldName string `json:"old_name"`
+	NewName string `json:"new_name"`
+}
+
 // CopyFileRequest is the request structure of the corresponding request
 type CopyFileRequest struct {
 	URLs     []string `json:"urls"`
@@ -84,7 +101,7 @@ type CopyFileResponse struct {
 	URLs    []FileCopy `json:"urls"`
 }
 
-// FileCopy is used in the the CopyFileResponse
+// FileCopy is used in the CopyFileResponse
 type FileCopy struct {
 	FromURL string `json:"from_url"`
 	ToURL   string `json:"to_url"`

backend/filefabric/api/types.go

@@ -19,7 +19,7 @@ const (
 	timeFormatJSON = `"` + timeFormatParameters + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // filefabric API
 type Time time.Time
 
@@ -95,7 +95,7 @@ type Status struct {
 	// Warning string `json:"warning"` // obsolete
 }
 
-// Status statisfies the error interface
+// Status satisfies the error interface
 func (e *Status) Error() string {
 	return fmt.Sprintf("%s (%s)", e.Message, e.Code)
 }

backend/filefabric/filefabric.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -85,6 +84,7 @@ Leave blank normally.
 
 Fill in to make rclone start with directory of a given ID.
 `,
+			Sensitive: true,
 		}, {
 			Name: "permanent_token",
 			Help: `Permanent Authentication Token.
@@ -98,6 +98,7 @@ These tokens are normally valid for several years.
 
 For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens
 `,
+			Sensitive: true,
 		}, {
 			Name: "token",
 			Help: `Session Token.
@@ -107,7 +108,8 @@ usually valid for 1 hour.
 
 Don't set this value - rclone will set it automatically.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "token_expiry",
 			Help: `Token expiry time.
@@ -150,7 +152,7 @@ type Fs struct {
 	opt             Options            // parsed options
 	features        *fs.Features       // optional features
 	m               configmap.Mapper   // to save config
-	srv             *rest.Client       // the connection to the one drive server
+	srv             *rest.Client       // the connection to the server
 	dirCache        *dircache.DirCache // Map of directory path to directory id
 	pacer           *fs.Pacer          // pacer for API calls
 	tokenMu         sync.Mutex         // hold when reading the token
@@ -373,7 +375,7 @@ type params map[string]interface{}
 
 // rpc calls the rpc.php method of the SME file fabric
 //
-// This is an entry point to all the method calls
+// This is an entry point to all the method calls.
 //
 // If result is nil then resp.Body will need closing
 func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKError, options []fs.OpenOption) (resp *http.Response, err error) {
@@ -490,7 +492,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 				// Root is a dir - cache its ID
 				f.dirCache.Put(f.root, info.ID)
 			}
-		} else {
+			//} else {
 			// Root is not found so a directory
 		}
 	}
@@ -678,7 +680,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -697,7 +699,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -783,9 +785,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -843,7 +845,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.purgeCheck(ctx, dir, false)
 }
 
-// Wait for the the background task to complete if necessary
+// Wait for the background task to complete if necessary
 func (f *Fs) waitForBackgroundTask(ctx context.Context, taskID api.String) (err error) {
 	if taskID == "" || taskID == "0" {
 		// No task to wait for
@@ -956,9 +958,9 @@ func (f *Fs) move(ctx context.Context, isDir bool, id, oldLeaf, newLeaf, oldDire
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1135,7 +1137,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1187,7 +1188,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{
@@ -1201,7 +1202,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/ftp/ftp.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/proxy"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -45,16 +46,18 @@ const (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "ftp",
-		Description: "FTP Connection",
+		Description: "FTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "host",
-			Help:     "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
-			Required: true,
+			Name:      "host",
+			Help:      "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:    "user",
-			Help:    "FTP username.",
-			Default: currentUser,
+			Name:      "user",
+			Help:      "FTP username.",
+			Default:   currentUser,
+			Sensitive: true,
 		}, {
 			Name:    "port",
 			Help:    "FTP port number.",
@@ -70,7 +73,7 @@ func init() {
 When using implicit FTP over TLS the client connects using TLS
 right from the start which breaks compatibility with
 non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTP.`,
+than port 21. Cannot be used in combination with explicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "explicit_tls",
@@ -78,11 +81,25 @@ than port 21. Cannot be used in combination with explicit FTP.`,
 
 When using explicit FTP over TLS the client explicitly requests
 security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTP.`,
+to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
-			Name:     "concurrency",
-			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited.",
+			Name: "concurrency",
+			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+
+Note that setting this is very likely to cause deadlocks so it should
+be used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of |--transfers| and |--checkers|.
+
+If you use |--check-first| then it just needs to be one more than the
+maximum of |--checkers| and |--transfers|.
+
+So for |concurrency 3| you'd use |--checkers 2 --transfers 2
+--check-first| or |--checkers 1 --transfers 1|.
+
+`, "|", "`", -1),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -100,11 +117,21 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "disable_utf8",
+			Help:     "Disable using UTF-8 even if server advertises support.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "writing_mdtm",
 			Help:     "Use MDTM to set modification time (VsFtpd quirk)",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "force_list_hidden",
+			Help:     "Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -148,6 +175,18 @@ Enabled by default. Use 0 to disable.`,
 If this is set and no password is supplied then rclone will ask for a password
 `,
 			Advanced: true,
+		}, {
+			Name:    "socks_proxy",
+			Default: "",
+			Help: `Socks 5 proxy host.
+		
+		Supports the format user:pass@host:port, user@host:port, host:port.
+		
+		Example:
+		
+			myUser:myPass@localhost:9005
+		`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -184,12 +223,15 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
+	DisableUTF8       bool                 `config:"disable_utf8"`
 	WritingMDTM       bool                 `config:"writing_mdtm"`
+	ForceListHidden   bool                 `config:"force_list_hidden"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
+	SocksProxy        string               `config:"socks_proxy"`
 }
 
 // Fs represents a remote FTP server
@@ -289,18 +331,33 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+// Return a *textproto.Error if err contains one or nil otherwise
+func textprotoError(err error) (errX *textproto.Error) {
+	if errors.As(err, &errX) {
+		return errX
+	}
+	return nil
+}
+
+// returns true if this FTP error should be retried
+func isRetriableFtpError(err error) bool {
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
+			return true
+		}
+	}
+	return false
+}
+
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
+	if isRetriableFtpError(err) {
+		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -310,14 +367,49 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
-		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
+		baseDialer := fshttp.NewDialer(ctx)
+		if f.opt.SocksProxy != "" {
+			conn, err = proxy.SOCKS5Dial(network, address, f.opt.SocksProxy, baseDialer)
+		} else {
+			conn, err = baseDialer.Dial(network, address)
 		}
-		return
+		if err != nil {
+			return nil, err
+		}
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -325,12 +417,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -338,12 +424,18 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.DisableMLSD {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
 	}
+	if f.opt.DisableUTF8 {
+		ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
+	}
 	if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
 		ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
 	}
 	if f.opt.WritingMDTM {
 		ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
 	}
+	if f.opt.ForceListHidden {
+		ftpConfig = append(ftpConfig, ftp.DialWithForceListHidden(true))
+	}
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
@@ -407,8 +499,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
-		var tpErr *textproto.Error
-		if !errors.As(err, &tpErr) {
+		if tpErr := textprotoError(err); tpErr != nil {
 			nopErr := c.NoOp()
 			if nopErr != nil {
 				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
@@ -478,7 +569,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		protocol = "ftps://"
 	}
 	if opt.TLS && opt.ExplicitTLS {
-		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config")
 	}
 	var tlsConfig *tls.Config
 	if opt.TLS || opt.ExplicitTLS {
@@ -510,6 +601,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
+		PartialUploads:          true,
 	}).Fill(ctx, f)
 	// set the pool drainer timer going
 	if f.opt.IdleTimeout > 0 {
@@ -557,8 +649,7 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
@@ -569,8 +660,7 @@ func translateErrorFile(err error) error {
 
 // translateErrorDir turns FTP errors into rclone errors if possible for a directory
 func translateErrorDir(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
@@ -601,8 +691,7 @@ func (f *Fs) dirFromStandardPath(dir string) string {
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	fullPath := path.Join(f.root, remote)
-	if fullPath == "" || fullPath == "." || fullPath == "/" {
+	if remote == "" || remote == "." || remote == "/" {
 		// if root, assume exists and synthesize an entry
 		return &ftp.Entry{
 			Name: "",
@@ -610,13 +699,38 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 			Time: time.Now(),
 		}, nil
 	}
-	dir := path.Dir(fullPath)
-	base := path.Base(fullPath)
 
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("findItem: %w", err)
 	}
+
+	// returns TRUE if MLST is supported which is required to call GetEntry
+	if c.IsTimePreciseInList() {
+		entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
+		f.putFtpConnection(&c, err)
+		if err != nil {
+			err = translateErrorFile(err)
+			if err == fs.ErrorObjectNotFound {
+				return nil, nil
+			}
+			if errX := textprotoError(err); errX != nil {
+				switch errX.Code {
+				case ftp.StatusBadArguments:
+					err = nil
+				}
+			}
+			return nil, err
+		}
+		if entry != nil {
+			f.entryToStandard(entry)
+		}
+		return entry, nil
+	}
+
+	dir := path.Dir(remote)
+	base := path.Base(remote)
+
 	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -635,7 +749,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return nil, err
 	}
@@ -657,7 +771,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 
 // dirExists checks the directory pointed to by remote exists or not
 func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return false, fmt.Errorf("dirExists: %w", err)
 	}
@@ -709,7 +823,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	case <-timer.C:
 		// if timer fired assume no error but connection dead
 		fs.Errorf(f, "Timeout when waiting for List")
-		return nil, errors.New("Timeout when waiting for List")
+		return nil, errors.New("timeout when waiting for List")
 	}
 
 	// Annoyingly FTP returns success for a directory which
@@ -760,11 +874,12 @@ func (f *Fs) Hashes() hash.Set {
 
 // Precision shows whether modified time is supported or not depending on the
 // FTP server capabilities, namely whether FTP server:
-// - accepts the MDTM command to get file time (fGetTime)
-//   or supports MLSD returning precise file time in the list (fLstTime)
-// - accepts the MFMT command to set file time (fSetTime)
-//   or non-standard form of the MDTM command (fSetTime, too)
-//   used by VsFtpd for the same purpose (WritingMDTM)
+//   - accepts the MDTM command to get file time (fGetTime)
+//     or supports MLSD returning precise file time in the list (fLstTime)
+//   - accepts the MFMT command to set file time (fSetTime)
+//     or non-standard form of the MDTM command (fSetTime, too)
+//     used by VsFtpd for the same purpose (WritingMDTM)
+//
 // See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
 func (f *Fs) Precision() time.Duration {
 	if (f.fGetTime || f.fLstTime) && f.fSetTime {
@@ -800,32 +915,18 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // getInfo reads the FileInfo for a path
 func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	dir := path.Dir(remote)
-	base := path.Base(remote)
-
-	c, err := f.getFtpConnection(ctx)
+	file, err := f.findItem(ctx, remote)
 	if err != nil {
-		return nil, fmt.Errorf("getInfo: %w", err)
-	}
-	files, err := c.List(f.dirFromStandardPath(dir))
-	f.putFtpConnection(&c, err)
-	if err != nil {
-		return nil, translateErrorFile(err)
-	}
-
-	for i := range files {
-		file := files[i]
-		f.entryToStandard(file)
-		if file.Name == base {
-			info := &FileInfo{
-				Name:    remote,
-				Size:    file.Size,
-				ModTime: file.Time,
-				precise: f.fLstTime,
-				IsDir:   file.Type == ftp.EntryTypeFolder,
-			}
-			return info, nil
+		return nil, err
+	} else if file != nil {
+		info := &FileInfo{
+			Name:    remote,
+			Size:    file.Size,
+			ModTime: file.Time,
+			precise: f.fLstTime,
+			IsDir:   file.Type == ftp.EntryTypeFolder,
 		}
+		return info, nil
 	}
 	return nil, fs.ErrorObjectNotFound
 }
@@ -856,8 +957,7 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	}
 	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
@@ -1026,7 +1126,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 // SetModTime sets the modification time of the object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if !o.fs.fSetTime {
-		fs.Errorf(o.fs, "SetModTime is not supported")
+		fs.Debugf(o.fs, "SetModTime is not supported")
 		return nil
 	}
 	c, err := o.fs.getFtpConnection(ctx)
@@ -1098,8 +1198,7 @@ func (f *ftpReadCloser) Close() error {
 	// mask the error if it was caused by a premature close
 	// NB StatusAboutToSend is to work around a bug in pureftpd
 	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
@@ -1125,22 +1224,33 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+
+	var (
+		fd *ftp.Response
+		c  *ftp.ServerConn
+	)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		c, err = o.fs.getFtpConnection(ctx)
+		if err != nil {
+			return false, err // getFtpConnection has retries already
+		}
+		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+		if err != nil {
+			o.fs.putFtpConnection(&c, err)
+		}
+		return shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-	if err != nil {
-		o.fs.putFtpConnection(&c, err)
-		return nil, fmt.Errorf("open: %w", err)
-	}
+
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
@@ -1166,13 +1276,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	// Ignore error 250 here - send by some servers
-	if err != nil {
-		switch errX := err.(type) {
-		case *textproto.Error:
-			switch errX.Code {
-			case ftp.StatusRequestedFileActionOK:
-				err = nil
-			}
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusRequestedFileActionOK:
+			err = nil
 		}
 	}
 	if err != nil {

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000             // 100 MiB
-		idleTimeout = 40 * time.Millisecond // small because test server is local
-		maxTime     = 10 * time.Second      // prevent test hangup
+		fileSize    = 100000000        // 100 MiB
+		idleTimeout = 1 * time.Second  // small because test server is local
+		maxTime     = 10 * time.Second // prevent test hangup
 	)
 
 	if testing.Short() {

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,11 +19,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -43,6 +44,7 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
+	option "google.golang.org/api/option"
 
 	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
@@ -80,7 +82,8 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			if saFile != "" || saCreds != "" || anonymous == "true" {
+			envAuth, _ := m.Get("env_auth")
+			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -88,15 +91,21 @@ func init() {
 			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
-			Name: "project_number",
-			Help: "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+			Name:      "project_number",
+			Help:      "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+			Sensitive: true,
+		}, {
+			Name:      "user_project",
+			Help:      "User project.\n\nOptional - needed only for requester pays.",
+			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
-			Name: "service_account_credentials",
-			Help: "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
-			Hide: fs.OptionHideBoth,
+			Name:      "service_account_credentials",
+			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Hide:      fs.OptionHideBoth,
+			Sensitive: true,
 		}, {
 			Name:    "anonymous",
 			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",
@@ -295,6 +304,15 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
 			}},
+		}, {
+			Name:     "directory_markers",
+			Default:  false,
+			Advanced: true,
+			Help: `Upload an empty object with a trailing slash when a new directory is created
+
+Empty folders are unsupported for bucket based remotes, this option creates an empty
+object ending with "/", to persist the folder.
+`,
 		}, {
 			Name: "no_check_bucket",
 			Help: `If set, don't attempt to check the bucket exists or create it.
@@ -304,6 +322,23 @@ rclone does if you know the bucket exists already.
 `,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "decompress",
+			Help: `If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to GCS with "Content-Encoding: gzip"
+set. Normally rclone will download these files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be decompressed.
+`,
+			Advanced: true,
+			Default:  false,
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for the service.\n\nLeave blank normally.",
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -311,6 +346,17 @@ rclone does if you know the bucket exists already.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:    "env_auth",
+			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default: false,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 }
@@ -318,6 +364,7 @@ rclone does if you know the bucket exists already.
 // Options defines the configuration for this backend
 type Options struct {
 	ProjectNumber             string               `config:"project_number"`
+	UserProject               string               `config:"user_project"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
 	Anonymous                 bool                 `config:"anonymous"`
@@ -327,21 +374,26 @@ type Options struct {
 	Location                  string               `config:"location"`
 	StorageClass              string               `config:"storage_class"`
 	NoCheckBucket             bool                 `config:"no_check_bucket"`
+	Decompress                bool                 `config:"decompress"`
+	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
+	DirectoryMarkers          bool                 `config:"directory_markers"`
 }
 
 // Fs represents a remote storage server
 type Fs struct {
-	name          string           // name of this remote
-	root          string           // the path we are working on if any
-	opt           Options          // parsed options
-	features      *fs.Features     // optional features
-	svc           *storage.Service // the connection to the storage server
-	client        *http.Client     // authorized client
-	rootBucket    string           // bucket part of root (if any)
-	rootDirectory string           // directory part of root (if any)
-	cache         *bucket.Cache    // cache of bucket status
-	pacer         *fs.Pacer        // To pace the API calls
+	name           string           // name of this remote
+	root           string           // the path we are working on if any
+	opt            Options          // parsed options
+	features       *fs.Features     // optional features
+	svc            *storage.Service // the connection to the storage server
+	client         *http.Client     // authorized client
+	rootBucket     string           // bucket part of root (if any)
+	rootDirectory  string           // directory part of root (if any)
+	cache          *bucket.Cache    // cache of bucket status
+	pacer          *fs.Pacer        // To pace the API calls
+	warnCompressed sync.Once        // warn once about compressed files
 }
 
 // Object describes a storage object
@@ -355,6 +407,7 @@ type Object struct {
 	bytes    int64     // Bytes in the object
 	modTime  time.Time // Modified time of the object
 	mimeType string
+	gzipped  bool // set if object has Content-Encoding: gzip
 }
 
 // ------------------------------------------------------------
@@ -372,7 +425,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootBucket == "" {
-		return fmt.Sprintf("GCS root")
+		return "GCS root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("GCS bucket %s", f.rootBucket)
@@ -421,7 +474,7 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -464,7 +517,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -477,6 +530,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {
@@ -502,10 +560,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		BucketBased:       true,
 		BucketBasedRootOK: true,
 	}).Fill(ctx, f)
+	if opt.DirectoryMarkers {
+		f.features.CanHaveEmptyDirectories = true
+	}
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
-	f.svc, err = storage.New(f.client)
+	gcsOpts := []option.ClientOption{option.WithHTTPClient(f.client)}
+	if opt.Endpoint != "" {
+		gcsOpts = append(gcsOpts, option.WithEndpoint(opt.Endpoint))
+	}
+	f.svc, err = storage.NewService(context.Background(), gcsOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create Google Cloud Storage client: %w", err)
 	}
@@ -514,7 +579,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// Check to see if the object exists
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
+			get := f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx)
+			if f.opt.UserProject != "" {
+				get = get.UserProject(f.opt.UserProject)
+			}
+			_, err = get.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -562,7 +631,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 //
 // dir is the starting directory, "" for root
 //
-// Set recurse to read sub directories
+// Set recurse to read sub directories.
 //
 // The remote has prefix removed from it and if addBucket is set
 // then it adds the bucket to the start.
@@ -574,9 +643,13 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		directory += "/"
 	}
 	list := f.svc.Objects.List(bucket).Prefix(directory).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		list = list.UserProject(f.opt.UserProject)
+	}
 	if !recurse {
 		list = list.Delimiter("/")
 	}
+	foundItems := 0
 	for {
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
@@ -592,6 +665,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			return err
 		}
 		if !recurse {
+			foundItems += len(objects.Prefixes)
 			var object storage.Object
 			for _, remote := range objects.Prefixes {
 				if !strings.HasSuffix(remote, "/") {
@@ -612,22 +686,29 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				}
 			}
 		}
+		foundItems += len(objects.Items)
 		for _, object := range objects.Items {
 			remote := f.opt.Enc.ToStandardPath(object.Name)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", object.Name)
 				continue
 			}
-			remote = remote[len(prefix):]
 			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
+			// is this a directory marker?
+			if isDirectory {
+				// Don't insert the root directory
+				if remote == directory {
+					continue
+				}
+				// process directory markers as directories
+				remote = strings.TrimRight(remote, "/")
+			}
+			remote = remote[len(prefix):]
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
-			// is this a directory marker?
-			if isDirectory {
-				continue // skip directory marker
-			}
-			err = fn(remote, object, false)
+
+			err = fn(remote, object, isDirectory)
 			if err != nil {
 				return err
 			}
@@ -637,6 +718,17 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 		list.PageToken(objects.NextPageToken)
 	}
+	if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
+		// Determine whether the directory exists or not by whether it has a marker
+		_, err := f.readObjectInfo(ctx, bucket, directory)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound {
+				return fs.ErrorDirNotFound
+			}
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -680,6 +772,9 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		return nil, errors.New("can't list buckets without project number")
 	}
 	listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		listBuckets = listBuckets.UserProject(f.opt.UserProject)
+	}
 	for {
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
@@ -780,7 +875,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -797,10 +892,69 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	return f.Put(ctx, in, src, options...)
 }
 
+// Create directory marker file and parents
+func (f *Fs) createDirectoryMarker(ctx context.Context, bucket, dir string) error {
+	if !f.opt.DirectoryMarkers || bucket == "" {
+		return nil
+	}
+
+	// Object to be uploaded
+	o := &Object{
+		fs:      f,
+		modTime: time.Now(),
+	}
+
+	for {
+		_, bucketPath := f.split(dir)
+		// Don't create the directory marker if it is the bucket or at the very root
+		if bucketPath == "" {
+			break
+		}
+		o.remote = dir + "/"
+
+		// Check to see if object already exists
+		_, err := o.readObjectInfo(ctx)
+		if err == nil {
+			return nil
+		}
+
+		// Upload it if not
+		fs.Debugf(o, "Creating directory marker")
+		content := io.Reader(strings.NewReader(""))
+		err = o.Update(ctx, content, o)
+		if err != nil {
+			return fmt.Errorf("creating directory marker failed: %w", err)
+		}
+
+		// Now check parent directory exists
+		dir = path.Dir(dir)
+		if dir == "/" || dir == "." {
+			break
+		}
+	}
+
+	return nil
+}
+
 // Mkdir creates the bucket if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 	bucket, _ := f.split(dir)
-	return f.makeBucket(ctx, bucket)
+	e := f.checkBucket(ctx, bucket)
+	if e != nil {
+		return e
+	}
+	return f.createDirectoryMarker(ctx, bucket, dir)
+
+}
+
+// mkdirParent creates the parent bucket/directory if it doesn't exist
+func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
+	remote = strings.TrimRight(remote, "/")
+	dir := path.Dir(remote)
+	if dir == "/" || dir == "." {
+		dir = ""
+	}
+	return f.Mkdir(ctx, dir)
 }
 
 // makeBucket creates the bucket if it doesn't exist
@@ -809,7 +963,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
+			list := f.svc.Objects.List(bucket).MaxResults(1).Context(ctx)
+			if f.opt.UserProject != "" {
+				list = list.UserProject(f.opt.UserProject)
+			}
+			_, err = list.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -844,7 +1002,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 			if !f.opt.BucketPolicyOnly {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
-			_, err = insertBucket.Context(ctx).Do()
+			insertBucket = insertBucket.Context(ctx)
+			if f.opt.UserProject != "" {
+				insertBucket = insertBucket.UserProject(f.opt.UserProject)
+			}
+			_, err = insertBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	}, nil)
@@ -864,12 +1026,28 @@ func (f *Fs) checkBucket(ctx context.Context, bucket string) error {
 // to delete was not empty.
 func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	bucket, directory := f.split(dir)
+	// Remove directory marker file
+	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
+		o := &Object{
+			fs:     f,
+			remote: dir + "/",
+		}
+		fs.Debugf(o, "Removing directory marker")
+		err := o.Remove(ctx)
+		if err != nil {
+			return fmt.Errorf("removing directory marker failed: %w", err)
+		}
+	}
 	if bucket == "" || directory != "" {
 		return nil
 	}
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
-			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
+			deleteBucket := f.svc.Buckets.Delete(bucket).Context(ctx)
+			if f.opt.UserProject != "" {
+				deleteBucket = deleteBucket.UserProject(f.opt.UserProject)
+			}
+			err = deleteBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	})
@@ -882,16 +1060,16 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	dstBucket, dstPath := f.split(remote)
-	err := f.checkBucket(ctx, dstBucket)
+	err := f.mkdirParent(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
@@ -915,7 +1093,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var rewriteResponse *storage.RewriteResponse
 	for {
 		err = f.pacer.Call(func() (bool, error) {
-			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
+			rewriteRequest = rewriteRequest.Context(ctx)
+			if f.opt.UserProject != "" {
+				rewriteRequest.UserProject(f.opt.UserProject)
+			}
+			rewriteResponse, err = rewriteRequest.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -975,6 +1157,7 @@ func (o *Object) setMetaData(info *storage.Object) {
 	o.url = info.MediaLink
 	o.bytes = int64(info.Size)
 	o.mimeType = info.ContentType
+	o.gzipped = info.ContentEncoding == "gzip"
 
 	// Read md5sum
 	md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@@ -1013,13 +1196,28 @@ func (o *Object) setMetaData(info *storage.Object) {
 	} else {
 		o.modTime = modTime
 	}
+
+	// If gunzipping then size and md5sum are unknown
+	if o.gzipped && o.fs.opt.Decompress {
+		o.bytes = -1
+		o.md5sum = ""
+	}
 }
 
 // readObjectInfo reads the definition for an object
 func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, err error) {
 	bucket, bucketPath := o.split()
-	err = o.fs.pacer.Call(func() (bool, error) {
-		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
+	return o.fs.readObjectInfo(ctx, bucket, bucketPath)
+}
+
+// readObjectInfo reads the definition for an object
+func (f *Fs) readObjectInfo(ctx context.Context, bucket, bucketPath string) (object *storage.Object, err error) {
+	err = f.pacer.Call(func() (bool, error) {
+		get := f.svc.Objects.Get(bucket, bucketPath).Context(ctx)
+		if f.opt.UserProject != "" {
+			get = get.UserProject(f.opt.UserProject)
+		}
+		object, err = get.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1091,7 +1289,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 		if !o.fs.opt.BucketPolicyOnly {
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = copyObject.Context(ctx).Do()
+		copyObject = copyObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			copyObject = copyObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = copyObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1108,11 +1310,26 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.fs.opt.UserProject != "" {
+		o.url = o.url + "&userProject=" + o.fs.opt.UserProject
+	}
 	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
 	if err != nil {
 		return nil, err
 	}
 	fs.FixRangeOption(options, o.bytes)
+	if o.gzipped && !o.fs.opt.Decompress {
+		// Allow files which are stored on the cloud storage system
+		// compressed to be downloaded without being decompressed.  Note
+		// that setting this here overrides the automatic decompression
+		// in the Transport.
+		//
+		// See: https://cloud.google.com/storage/docs/transcoding
+		req.Header.Set("Accept-Encoding", "gzip")
+		o.fs.warnCompressed.Do(func() {
+			fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-decompress to override")
+		})
+	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -1139,11 +1356,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	err := o.fs.checkBucket(ctx, bucket)
-	if err != nil {
-		return err
+	// Create parent dir/bucket if not saving directory marker
+	if !strings.HasSuffix(o.remote, "/") {
+		err = o.fs.mkdirParent(ctx, o.remote)
+		if err != nil {
+			return err
+		}
 	}
 	modTime := src.ModTime(ctx)
 
@@ -1188,7 +1408,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if !o.fs.opt.BucketPolicyOnly {
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = insertObject.Context(ctx).Do()
+		insertObject = insertObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			insertObject = insertObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = insertObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1203,7 +1427,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
+		deleteBucket := o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			deleteBucket = deleteBucket.UserProject(o.fs.opt.UserProject)
+		}
+		err = deleteBucket.Do()
 		return shouldRetry(ctx, err)
 	})
 	return err

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/googlecloudstorage"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -16,3 +17,17 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*googlecloudstorage.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	name := "TestGoogleCloudStorage"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*googlecloudstorage.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "directory_markers", Value: "true"},
+		},
+	})
+}

backend/googlephotos/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Google Photos API.
 package api
 
 import (

backend/googlephotos/googlephotos.go

@@ -178,7 +178,7 @@ type Fs struct {
 	opt        Options                // parsed options
 	features   *fs.Features           // optional features
 	unAuth     *rest.Client           // unauthenticated http client
-	srv        *rest.Client           // the connection to the one drive server
+	srv        *rest.Client           // the connection to the server
 	ts         *oauthutil.TokenSource // token source for oauth2
 	pacer      *fs.Pacer              // To pace the API calls
 	startTime  time.Time              // time Fs was started - used for datestamps
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
 		for i := range items {
 			item := &result.MediaItems[i]
 			remote := item.Filename
-			remote = strings.Replace(remote, "/", "／", -1)
+			remote = strings.ReplaceAll(remote, "/", "／")
 			err = fn(remote, item, false)
 			if err != nil {
 				return err
@@ -661,7 +661,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {

backend/googlephotos/googlephotos_test.go

@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"testing"
@@ -12,7 +12,6 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -37,7 +36,7 @@ func TestIntegration(t *testing.T) {
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
 	if err == fs.ErrorNotFoundInConfigFile {
-		t.Skip(fmt.Sprintf("Couldn't create google photos backend - skipping tests: %v", err))
+		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)
 
@@ -56,7 +55,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+			dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -99,7 +98,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := ioutil.ReadAll(in)
+				buf, err := io.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)
@@ -221,7 +220,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+		dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/googlephotos/pattern.go

@@ -315,7 +315,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 
 // featureFilter creates a filter for the Feature enum
 //
-// The API only supports one feature, FAVORITES, so hardcode that feature
+// The API only supports one feature, FAVORITES, so hardcode that feature.
 //
 // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
 func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {

backend/googlephotos/pattern_test.go

@@ -50,7 +50,7 @@ func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums,
 
 // mock listUploads for testing
 func (f *testLister) listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entries, _ = f.uploaded[dir]
+	entries = f.uploaded[dir]
 	return entries, nil
 }
 

backend/hasher/commands.go

@@ -161,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
 			if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
 				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
 			}
-			accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
+			accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
 			doneCount++
 		}
 	})

backend/hasher/hasher.go

@@ -27,6 +27,9 @@ func init() {
 		Name:        "hasher",
 		Description: "Better checksums for other remotes",
 		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
@@ -158,6 +161,12 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 		IsLocal:                 true,
 		ReadMimeType:            true,
 		WriteMimeType:           true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -282,7 +291,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	if do := f.Fs.Features().CleanUp; do != nil {
 		return do(ctx)
 	}
-	return errors.New("CleanUp not supported")
+	return errors.New("not supported by underlying remote")
 }
 
 // About gets quota information from the Fs
@@ -290,7 +299,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	if do := f.Fs.Features().About; do != nil {
 		return do(ctx)
 	}
-	return nil, errors.New("About not supported")
+	return nil, errors.New("not supported by underlying remote")
 }
 
 // ChangeNotify calls the passed function with a path that has had changes.
@@ -485,6 +494,17 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return ""
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.Object.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -507,10 +527,5 @@ var (
 	_ fs.UserInfoer      = (*Fs)(nil)
 	_ fs.Disconnecter    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
-	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/hasher/hasher_internal_test.go

@@ -19,7 +19,7 @@ import (
 func putFile(ctx context.Context, t *testing.T, f fs.Fs, name, data string) fs.Object {
 	mtime1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	item := fstest.Item{Path: name, ModTime: mtime1}
-	_, o := fstests.PutTestContents(ctx, t, f, &item, data, true)
+	o := fstests.PutTestContents(ctx, t, f, &item, data, true)
 	require.NotNil(t, o)
 	return o
 }
@@ -35,7 +35,7 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
 	// make a temporary crypt remote
 	ctx := context.Background()
 	pass := obscure.MustObscure("crypt")
-	remote := fmt.Sprintf(":crypt,remote=%s,password=%s:", tempRoot, pass)
+	remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass)
 	cryptFs, err := fs.NewFs(ctx, remote)
 	require.NoError(t, err)
 

backend/hasher/hasher_test.go

@@ -33,6 +33,7 @@ func TestIntegration(t *testing.T) {
 			{Name: "TestHasher", Key: "remote", Value: tempDir},
 		}
 		opt.RemoteName = "TestHasher:"
+		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
 }

backend/hasher/object.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"time"
 
@@ -118,7 +117,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(ioutil.Discard, r); err != nil {
+	if _, err = io.Copy(io.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}

backend/hdfs/fs.go

@@ -92,7 +92,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.ServicePrincipalName != "" {
 		options.KerberosClient, err = getKerberosClient()
 		if err != nil {
-			return nil, fmt.Errorf("Problem with kerberos authentication: %s", err)
+			return nil, fmt.Errorf("problem with kerberos authentication: %w", err)
 		}
 		options.KerberosServicePrincipleName = opt.ServicePrincipalName
 
@@ -265,9 +265,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/hdfs/hdfs.go

@@ -1,6 +1,7 @@
 //go:build !plan9
 // +build !plan9
 
+// Package hdfs provides an interface to the HDFS storage system.
 package hdfs
 
 import (
@@ -18,9 +19,10 @@ func init() {
 		Description: "Hadoop distributed file system",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "namenode",
-			Help:     "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
-			Required: true,
+			Name:      "namenode",
+			Help:      "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name: "username",
 			Help: "Hadoop user name.",
@@ -28,6 +30,7 @@ func init() {
 				Value: "root",
 				Help:  "Connect to hdfs as root.",
 			}},
+			Sensitive: true,
 		}, {
 			Name: "service_principal_name",
 			Help: `Kerberos service principal name for the namenode.
@@ -35,15 +38,16 @@ func init() {
 Enables KERBEROS authentication. Specifies the Service Principal Name
 (SERVICE/FQDN) for the namenode. E.g. \"hdfs/namenode.hadoop.docker\"
 for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "data_transfer_protection",
 			Help: `Kerberos data transfer protection: authentication|integrity|privacy.
 
 Specifies whether or not authentication, data signature integrity
-checks, and wire encryption is required when communicating the the
-datanodes. Possible values are 'authentication', 'integrity' and
-'privacy'. Used only with KERBEROS enabled.`,
+checks, and wire encryption are required when communicating with
+the datanodes. Possible values are 'authentication', 'integrity'
+and 'privacy'. Used only with KERBEROS enabled.`,
 			Examples: []fs.OptionExample{{
 				Value: "privacy",
 				Help:  "Ensure authentication, integrity and encryption enabled.",

backend/hdfs/object.go

@@ -115,7 +115,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	info, err := o.fs.client.Stat(realpath)
+	_, err = o.fs.client.Stat(realpath)
 	if err == nil {
 		err = o.fs.client.Remove(realpath)
 		if err != nil {
@@ -147,7 +147,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	info, err = o.fs.client.Stat(realpath)
+	info, err := o.fs.client.Stat(realpath)
 	if err != nil {
 		return err
 	}

backend/hidrive/api/queries.go

@@ -0,0 +1,81 @@
+package api
+
+import (
+	"encoding/json"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+// Some presets for different amounts of information that can be requested for fields;
+// it is recommended to only request the information that is actually needed.
+var (
+	HiDriveObjectNoMetadataFields            = []string{"name", "type"}
+	HiDriveObjectWithMetadataFields          = append(HiDriveObjectNoMetadataFields, "id", "size", "mtime", "chash")
+	HiDriveObjectWithDirectoryMetadataFields = append(HiDriveObjectWithMetadataFields, "nmembers")
+	DirectoryContentFields                   = []string{"nmembers"}
+)
+
+// QueryParameters represents the parameters passed to an API-call.
+type QueryParameters struct {
+	url.Values
+}
+
+// NewQueryParameters initializes an instance of QueryParameters and
+// returns a pointer to it.
+func NewQueryParameters() *QueryParameters {
+	return &QueryParameters{url.Values{}}
+}
+
+// SetFileInDirectory sets the appropriate parameters
+// to specify a path to a file in a directory.
+// This is used by requests that work with paths for files that do not exist yet.
+// (For example when creating a file).
+// Most requests use the format produced by SetPath(...).
+func (p *QueryParameters) SetFileInDirectory(filePath string) {
+	directory, file := path.Split(path.Clean(filePath))
+	p.Set("dir", path.Clean(directory))
+	p.Set("name", file)
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetPath sets the appropriate parameters to access the given path.
+func (p *QueryParameters) SetPath(objectPath string) {
+	p.Set("path", path.Clean(objectPath))
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetTime sets the key to the time-value. It replaces any existing values.
+func (p *QueryParameters) SetTime(key string, value time.Time) error {
+	valueAPI := Time(value)
+	valueBytes, err := json.Marshal(&valueAPI)
+	if err != nil {
+		return err
+	}
+	p.Set(key, string(valueBytes))
+	return nil
+}
+
+// AddList adds the given values as a list
+// with each value separated by the separator.
+// It appends to any existing values associated with key.
+func (p *QueryParameters) AddList(key string, separator string, values ...string) {
+	original := p.Get(key)
+	p.Set(key, strings.Join(values, separator))
+	if original != "" {
+		p.Set(key, original+separator+p.Get(key))
+	}
+}
+
+// AddFields sets the appropriate parameter to access the given fields.
+// The given fields will be appended to any other existing fields.
+func (p *QueryParameters) AddFields(prefix string, fields ...string) {
+	modifiedFields := make([]string, len(fields))
+	for i, field := range fields {
+		modifiedFields[i] = prefix + field
+	}
+	p.AddList("fields", ",", modifiedFields...)
+}

backend/hidrive/api/types.go

@@ -0,0 +1,135 @@
+// Package api has type definitions and code related to API-calls for the HiDrive-API.
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// Time represents date and time information for the API.
+type Time time.Time
+
+// MarshalJSON turns Time into JSON (in Unix-time/UTC).
+func (t *Time) MarshalJSON() ([]byte, error) {
+	secs := time.Time(*t).Unix()
+	return []byte(strconv.FormatInt(secs, 10)), nil
+}
+
+// UnmarshalJSON turns JSON into Time.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	secs, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = Time(time.Unix(secs, 0))
+	return nil
+}
+
+// Error is returned from the API when things go wrong.
+type Error struct {
+	Code        json.Number `json:"code"`
+	ContextInfo json.RawMessage
+	Message     string `json:"msg"`
+}
+
+// Error returns a string for the error and satisfies the error interface.
+func (e *Error) Error() string {
+	out := fmt.Sprintf("Error %q", e.Code.String())
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.ContextInfo != nil {
+		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
+	}
+	return out
+}
+
+// Check Error satisfies the error interface.
+var _ error = (*Error)(nil)
+
+// possible types for HiDriveObject
+const (
+	HiDriveObjectTypeDirectory = "dir"
+	HiDriveObjectTypeFile      = "file"
+	HiDriveObjectTypeSymlink   = "symlink"
+)
+
+// HiDriveObject describes a folder, a symlink or a file.
+// Depending on the type and content, not all fields are present.
+type HiDriveObject struct {
+	Type         string `json:"type"`
+	ID           string `json:"id"`
+	ParentID     string `json:"parent_id"`
+	Name         string `json:"name"`
+	Path         string `json:"path"`
+	Size         int64  `json:"size"`
+	MemberCount  int64  `json:"nmembers"`
+	ModifiedAt   Time   `json:"mtime"`
+	ChangedAt    Time   `json:"ctime"`
+	MetaHash     string `json:"mhash"`
+	MetaOnlyHash string `json:"mohash"`
+	NameHash     string `json:"nhash"`
+	ContentHash  string `json:"chash"`
+	IsTeamfolder bool   `json:"teamfolder"`
+	Readable     bool   `json:"readable"`
+	Writable     bool   `json:"writable"`
+	Shareable    bool   `json:"shareable"`
+	MIMEType     string `json:"mime_type"`
+}
+
+// ModTime returns the modification time of the HiDriveObject.
+func (i *HiDriveObject) ModTime() time.Time {
+	t := time.Time(i.ModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.ChangedAt)
+	}
+	return t
+}
+
+// UnmarshalJSON turns JSON into HiDriveObject and
+// introduces specific default-values where necessary.
+func (i *HiDriveObject) UnmarshalJSON(data []byte) error {
+	type objectAlias HiDriveObject
+	defaultObject := objectAlias{
+		Size:        -1,
+		MemberCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultObject)
+	if err != nil {
+		return err
+	}
+	name, err := url.PathUnescape(defaultObject.Name)
+	if err == nil {
+		defaultObject.Name = name
+	}
+
+	*i = HiDriveObject(defaultObject)
+	return nil
+}
+
+// DirectoryContent describes the content of a directory.
+type DirectoryContent struct {
+	TotalCount int64           `json:"nmembers"`
+	Entries    []HiDriveObject `json:"members"`
+}
+
+// UnmarshalJSON turns JSON into DirectoryContent and
+// introduces specific default-values where necessary.
+func (d *DirectoryContent) UnmarshalJSON(data []byte) error {
+	type directoryContentAlias DirectoryContent
+	defaultDirectoryContent := directoryContentAlias{
+		TotalCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultDirectoryContent)
+	if err != nil {
+		return err
+	}
+
+	*d = DirectoryContent(defaultDirectoryContent)
+	return nil
+}

backend/hidrive/helpers.go

@@ -0,0 +1,879 @@
+package hidrive
+
+// This file is for helper-functions which may provide more general and
+// specialized functionality than the generic interfaces.
+// There are two sections:
+// 1. methods bound to Fs
+// 2. other functions independent from Fs used throughout the package
+
+// NOTE: Functions accessing paths expect any relative paths
+// to be resolved prior to execution with resolvePath(...).
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/hidrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/ranges"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+)
+
+const (
+	// MaximumUploadBytes represents the maximum amount of bytes
+	// a single upload-operation will support.
+	MaximumUploadBytes = 2147483647 // = 2GiB - 1
+	// iterationChunkSize represents the chunk size used to iterate directory contents.
+	iterationChunkSize = 5000
+)
+
+var (
+	// retryErrorCodes is a slice of error codes that we will always retry.
+	retryErrorCodes = []int{
+		429, // Too Many Requests
+		500, // Internal Server Error
+		502, // Bad Gateway
+		503, // Service Unavailable
+		504, // Gateway Timeout
+		509, // Bandwidth Limit Exceeded
+	}
+	// ErrorFileExists is returned when a query tries to create a file
+	// that already exists.
+	ErrorFileExists = errors.New("destination file already exists")
+)
+
+// MemberType represents the possible types of entries a directory can contain.
+type MemberType string
+
+// possible values for MemberType
+const (
+	AllMembers       MemberType = "all"
+	NoMembers        MemberType = "none"
+	DirectoryMembers MemberType = api.HiDriveObjectTypeDirectory
+	FileMembers      MemberType = api.HiDriveObjectTypeFile
+	SymlinkMembers   MemberType = api.HiDriveObjectTypeSymlink
+)
+
+// SortByField represents possible fields to sort entries of a directory by.
+type SortByField string
+
+// possible values for SortByField
+const (
+	descendingSort             string      = "-"
+	SortByName                 SortByField = "name"
+	SortByModTime              SortByField = "mtime"
+	SortByObjectType           SortByField = "type"
+	SortBySize                 SortByField = "size"
+	SortByNameDescending       SortByField = SortByField(descendingSort) + SortByName
+	SortByModTimeDescending    SortByField = SortByField(descendingSort) + SortByModTime
+	SortByObjectTypeDescending SortByField = SortByField(descendingSort) + SortByObjectType
+	SortBySizeDescending       SortByField = SortByField(descendingSort) + SortBySize
+)
+
+var (
+	// Unsorted disables sorting and can therefore not be combined with other values.
+	Unsorted = []SortByField{"none"}
+	// DefaultSorted does not specify how to sort and
+	// therefore implies the default sort order.
+	DefaultSorted = []SortByField{}
+)
+
+// CopyOrMoveOperationType represents the possible types of copy- and move-operations.
+type CopyOrMoveOperationType int
+
+// possible values for CopyOrMoveOperationType
+const (
+	MoveOriginal CopyOrMoveOperationType = iota
+	CopyOriginal
+	CopyOriginalPreserveModTime
+)
+
+// OnExistAction represents possible actions the API should take,
+// when a request tries to create a path that already exists.
+type OnExistAction string
+
+// possible values for OnExistAction
+const (
+	// IgnoreOnExist instructs the API not to execute
+	// the request in case of a conflict, but to return an error.
+	IgnoreOnExist OnExistAction = "ignore"
+	// AutoNameOnExist instructs the API to automatically rename
+	// any conflicting request-objects.
+	AutoNameOnExist OnExistAction = "autoname"
+	// OverwriteOnExist instructs the API to overwrite any conflicting files.
+	// This can only be used, if the request operates on files directly.
+	// (For example when moving/copying a file.)
+	// For most requests this action will simply be ignored.
+	OverwriteOnExist OnExistAction = "overwrite"
+)
+
+// shouldRetry returns a boolean as to whether this resp and err deserve to be retried.
+// It tries to expire/invalidate the token, if necessary.
+// It returns the err as a convenience.
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	if resp != nil && (resp.StatusCode == 401 || isHTTPError(err, 401)) && len(resp.Header["Www-Authenticate"]) > 0 {
+		fs.Debugf(f, "Token might be invalid: %v", err)
+		if f.tokenRenewer != nil {
+			iErr := f.tokenRenewer.Expire()
+			if iErr == nil {
+				return true, err
+			}
+		}
+	}
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// resolvePath resolves the given (relative) path and
+// returns a path suitable for API-calls.
+// This will consider the root-path of the fs and any needed prefixes.
+//
+// Any relative paths passed to functions that access these paths should
+// be resolved with this first!
+func (f *Fs) resolvePath(objectPath string) string {
+	resolved := path.Join(f.opt.RootPrefix, f.root, f.opt.Enc.FromStandardPath(objectPath))
+	return resolved
+}
+
+// iterateOverDirectory calls the given function callback
+// on each item found in a given directory.
+//
+// If callback ever returns true then this exits early with found = true.
+func (f *Fs) iterateOverDirectory(ctx context.Context, directory string, searchOnly MemberType, callback func(*api.HiDriveObject) bool, fields []string, sortBy []SortByField) (found bool, err error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.AddFields("members.", fields...)
+	parameters.AddFields("", api.DirectoryContentFields...)
+	parameters.Set("members", string(searchOnly))
+	for _, v := range sortBy {
+		// The explicit conversion is necessary for each element.
+		parameters.AddList("sort", ",", string(v))
+	}
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	iterateContent := func(result *api.DirectoryContent, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+		for _, item := range result.Entries {
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
+			if callback(&item) {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+	return f.paginateDirectoryAccess(ctx, &opts, iterationChunkSize, 0, iterateContent)
+}
+
+// paginateDirectoryAccess executes requests specified via ctx and opts
+// which should produce api.DirectoryContent.
+// This will paginate the requests using limit starting at the given offset.
+//
+// The given function callback is called on each api.DirectoryContent found
+// along with any errors that occurred.
+// If callback ever returns true then this exits early with found = true.
+// If callback ever returns an error then this exits early with that error.
+func (f *Fs) paginateDirectoryAccess(ctx context.Context, opts *rest.Opts, limit int64, offset int64, callback func(*api.DirectoryContent, error) (bool, error)) (found bool, err error) {
+	for {
+		opts.Parameters.Set("limit", strconv.FormatInt(offset, 10)+","+strconv.FormatInt(limit, 10))
+
+		var result api.DirectoryContent
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, opts, nil, &result)
+			return f.shouldRetry(ctx, resp, err)
+		})
+
+		found, err = callback(&result, err)
+		if found || err != nil {
+			return found, err
+		}
+
+		offset += int64(len(result.Entries))
+		if offset >= result.TotalCount || limit > int64(len(result.Entries)) {
+			break
+		}
+	}
+	return false, nil
+}
+
+// fetchMetadataForPath reads the metadata from the path.
+func (f *Fs) fetchMetadataForPath(ctx context.Context, path string, fields []string) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.AddFields("", fields...)
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/meta",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// copyOrMove copies or moves a directory or file
+// from the source-path to the destination-path.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: Use the explicit methods instead of directly invoking this method.
+// (Those are: copyDirectory, moveDirectory, copyFile, moveFile.)
+func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType CopyOrMoveOperationType, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.Set("src", source)
+	parameters.Set("dst", destination)
+	if onExist == AutoNameOnExist ||
+		(onExist == OverwriteOnExist && !isDirectory) {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	endpoint := "/"
+	if isDirectory {
+		endpoint += "dir"
+	} else {
+		endpoint += "file"
+	}
+	switch operationType {
+	case MoveOriginal:
+		endpoint += "/move"
+	case CopyOriginalPreserveModTime:
+		parameters.Set("preserve_mtime", strconv.FormatBool(true))
+		fallthrough
+	case CopyOriginal:
+		endpoint += "/copy"
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       endpoint,
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// moveDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) moveDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, MoveOriginal, source, destination, onExist)
+}
+
+// copyFile copies the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation will expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) copyFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
+// moveFile moves the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation may expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) moveFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, MoveOriginal, source, destination, onExist)
+}
+
+// createDirectory creates the directory at the given path and
+// returns the resulting api-object if successful.
+//
+// The directory will only be created if its parent-directory exists.
+// This returns fs.ErrorDirNotFound if the parent-directory is not found.
+// This returns fs.ErrorDirExists if the directory already exists.
+func (f *Fs) createDirectory(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, fs.ErrorDirExists
+	}
+	return nil, err
+}
+
+// createDirectories creates the directory at the given path
+// along with any missing parent directories and
+// returns the resulting api-object (of the created directory) if successful.
+//
+// This returns fs.ErrorDirExists if the directory already exists.
+//
+// If an error occurs while the parent directories are being created,
+// any directories already created will NOT be deleted again.
+func (f *Fs) createDirectories(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	result, err := f.createDirectory(ctx, directory, onExist)
+	if err == nil {
+		return result, nil
+	}
+	if err != fs.ErrorDirNotFound {
+		return nil, err
+	}
+	parentDirectory := path.Dir(directory)
+	_, err = f.createDirectories(ctx, parentDirectory, onExist)
+	if err != nil && err != fs.ErrorDirExists {
+		return nil, err
+	}
+	// NOTE: Ignoring fs.ErrorDirExists does no harm,
+	// since it does not mean the child directory cannot be created.
+	return f.createDirectory(ctx, directory, onExist)
+}
+
+// deleteDirectory deletes the directory at the given path.
+//
+// If recursive is false, the directory will only be deleted if it is empty.
+// If recursive is true, the directory will be deleted regardless of its content.
+// This returns fs.ErrorDirNotFound if the directory is not found.
+// This returns fs.ErrorDirectoryNotEmpty if the directory is not empty and
+// recursive is false.
+func (f *Fs) deleteDirectory(ctx context.Context, directory string, recursive bool) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.Set("recursive", strconv.FormatBool(recursive))
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case isHTTPError(err, 404):
+		return fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return fs.ErrorDirectoryNotEmpty
+	}
+	return err
+}
+
+// deleteObject deletes the object/file at the given path.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) deleteObject(ctx context.Context, path string) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/file",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// createFile creates a file at the given path
+// with the content of the io.ReadSeeker.
+// This guarantees that existing files will not be overwritten.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+// This returns ErrorFileExists if a file already exists at the specified path.
+func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "POST",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, ErrorFileExists
+	}
+	return nil, err
+}
+
+// overwriteFile updates the content of the file at the given path
+// with the content of the io.ReadSeeker.
+// If the file does not exist it will be created.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PUT",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	}
+	return nil, err
+}
+
+// uploadFileChunked updates the content of the existing file at the given path
+// with the content of the io.Reader.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+// Returns the resulting api-object if successful.
+//
+// Replaces the file contents by uploading multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: This method uses updateFileChunked and may create sparse files,
+// if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) uploadFileChunked(ctx context.Context, path string, content io.Reader, modTime time.Time, chunkSize int, transferLimit int64) (okSize uint64, info *api.HiDriveObject, err error) {
+	okSize, err = f.updateFileChunked(ctx, path, content, 0, chunkSize, transferLimit)
+
+	if err == nil {
+		info, err = f.resizeFile(ctx, path, okSize, modTime)
+	}
+	return okSize, info, err
+}
+
+// updateFileChunked updates the content of the existing file at the given path
+// starting at the given offset.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.Reader.
+// If the offset is beyond the file end, the file is extended up to the offset.
+//
+// The upload is done multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+//
+// NOTE: Because it is inefficient to set the modification time with every chunk,
+// setting it to a specific value must be done in a separate request
+// after this operation finishes.
+//
+// NOTE: This method uses patchFile and may create sparse files,
+// especially if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Reader, offset uint64, chunkSize int, transferLimit int64) (okSize uint64, err error) {
+	var (
+		okChunksMu sync.Mutex // protects the variables below
+		okChunks   []ranges.Range
+	)
+	g, gCtx := errgroup.WithContext(ctx)
+	transferSemaphore := semaphore.NewWeighted(transferLimit)
+
+	var readErr error
+	startMoreTransfers := true
+	zeroTime := time.Time{}
+	for chunk := uint64(0); startMoreTransfers; chunk++ {
+		// Acquire semaphore to limit number of transfers in parallel.
+		readErr = transferSemaphore.Acquire(gCtx, 1)
+		if readErr != nil {
+			break
+		}
+
+		// Read a chunk of data.
+		chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
+		if bytesRead < chunkSize {
+			startMoreTransfers = false
+		}
+		if readErr != nil || bytesRead <= 0 {
+			break
+		}
+
+		// Transfer the chunk.
+		chunkOffset := uint64(chunkSize)*chunk + offset
+		g.Go(func() error {
+			// After this upload is done,
+			// signal that another transfer can be started.
+			defer transferSemaphore.Release(1)
+			uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
+			if uploadErr == nil {
+				// Remember successfully written chunks.
+				okChunksMu.Lock()
+				okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
+				okChunksMu.Unlock()
+				fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
+			} else {
+				fs.Infof(f, "Error while uploading chunk at offset %v. Error is %v.", chunkOffset, uploadErr)
+			}
+			return uploadErr
+		})
+	}
+
+	if readErr != nil {
+		// Log the error in case it is later ignored because of an upload-error.
+		fs.Infof(f, "Error while reading/preparing to upload a chunk. Error is %v.", readErr)
+	}
+
+	err = g.Wait()
+
+	// Compute the first continuous range of the file content,
+	// which does not contain any failed chunks.
+	// Do not forget to add the file content up to the starting offset,
+	// which is presumed to be already correct.
+	rs := ranges.Ranges{}
+	rs.Insert(ranges.Range{Pos: 0, Size: int64(offset)})
+	for _, chunkRange := range okChunks {
+		rs.Insert(chunkRange)
+	}
+	if len(rs) > 0 && rs[0].Pos == 0 {
+		okSize = uint64(rs[0].Size)
+	}
+
+	if err != nil {
+		return okSize, err
+	}
+	if readErr != nil {
+		return okSize, readErr
+	}
+
+	return okSize, nil
+}
+
+// patchFile updates the content of the existing file at the given path
+// starting at the given offset.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.ReadSeeker.
+// If the offset is beyond the file end, the file is extended up to the offset.
+// The maximum size of the update is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file up to the offset this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("offset", strconv.FormatUint(offset, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PATCH",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+		NoResponse:  true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		_, err = content.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		resp, err = f.srv.Call(ctx, &opts)
+		if isHTTPError(err, 423) {
+			return true, err
+		}
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// resizeFile updates the existing file at the given path to be of the given size
+// and returns the resulting api-object if successful.
+//
+// If the given size is smaller than the current filesize,
+// the file is cut/truncated at that position.
+// If the given size is larger, the file is extended up to that position.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) resizeFile(ctx context.Context, path string, size uint64, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("size", strconv.FormatUint(size, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/file/truncate",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorObjectNotFound
+	}
+	return nil, err
+}
+
+// ------------------------------------------------------------
+
+// isHTTPError compares the numerical status code
+// of an api.Error to the given HTTP status.
+//
+// If the given error is not an api.Error or
+// a numerical status code could not be determined, this returns false.
+// Otherwise this returns whether the status code of the error is equal to the given status.
+func isHTTPError(err error, status int64) bool {
+	if apiErr, ok := err.(*api.Error); ok {
+		errStatus, decodeErr := apiErr.Code.Int64()
+		if decodeErr == nil && errStatus == status {
+			return true
+		}
+	}
+	return false
+}
+
+// createHiDriveScopes creates oauth-scopes
+// from the given user-role and access-permissions.
+//
+// If the arguments are empty, they will not be included in the result.
+func createHiDriveScopes(role string, access string) []string {
+	switch {
+	case role != "" && access != "":
+		return []string{access + "," + role}
+	case role != "":
+		return []string{role}
+	case access != "":
+		return []string{access}
+	}
+	return []string{}
+}
+
+// cachedReader returns a version of the reader that caches its contents and
+// can therefore be reset using Seek.
+func cachedReader(reader io.Reader) io.ReadSeeker {
+	bytesReader, ok := reader.(*bytes.Reader)
+	if ok {
+		return bytesReader
+	}
+
+	repeatableReader, ok := reader.(*readers.RepeatableReader)
+	if ok {
+		return repeatableReader
+	}
+
+	return readers.NewRepeatableReader(reader)
+}
+
+// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
+// Returns a new io.Reader (chunkReader) for that chunk
+// and the number of bytes that have been read from reader.
+func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
+	// Unwrap any accounting from the input if present.
+	reader, wrap := accounting.UnWrap(reader)
+
+	// Read a chunk of data.
+	buffer := make([]byte, length)
+	bytesRead, err = io.ReadFull(reader, buffer)
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
+		err = nil
+	}
+	if err != nil {
+		return nil, bytesRead, err
+	}
+	// Truncate unused capacity.
+	buffer = buffer[:bytesRead]
+
+	// Use wrap to put any accounting back for chunkReader.
+	return wrap(bytes.NewReader(buffer)), bytesRead, nil
+}

backend/hidrive/hidrive_test.go

@@ -0,0 +1,45 @@
+// Test HiDrive filesystem interface
+package hidrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote.
+func TestIntegration(t *testing.T) {
+	name := "TestHiDrive"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize:       1,
+			MaxChunkSize:       MaximumUploadBytes,
+			CeilChunkSize:      nil,
+			NeedMultipleChunks: false,
+		},
+	})
+}
+
+// Change the configured UploadChunkSize.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadChunkSize(chunksize fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadChunkSize = f.opt.UploadChunkSize, chunksize
+	return old, nil
+}
+
+// Change the configured UploadCutoff.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
+	return old, nil
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)

backend/hidrive/hidrivehash/hidrivehash.go

@@ -0,0 +1,410 @@
+// Package hidrivehash implements the HiDrive hashing algorithm which combines SHA-1 hashes hierarchically to a single top-level hash.
+//
+// Note: This implementation does not grant access to any partial hashes generated.
+//
+// See: https://developer.hidrive.com/wp-content/uploads/2021/07/HiDrive_Synchronization-v3.3-rev28.pdf
+// (link to newest version: https://static.hidrive.com/dev/0001)
+package hidrivehash
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+)
+
+const (
+	// BlockSize of the checksum in bytes.
+	BlockSize = 4096
+	// Size of the checksum in bytes.
+	Size = sha1.Size
+	// sumsPerLevel is the number of checksums
+	sumsPerLevel = 256
+)
+
+var (
+	// zeroSum is a special hash consisting of 20 null-bytes.
+	// This will be the hash of any empty file (or ones containing only null-bytes).
+	zeroSum = [Size]byte{}
+	// ErrorInvalidEncoding is returned when a hash should be decoded from a binary form that is invalid.
+	ErrorInvalidEncoding = errors.New("encoded binary form is invalid for this hash")
+	// ErrorHashFull is returned when a hash reached its capacity and cannot accept any more input.
+	ErrorHashFull = errors.New("hash reached its capacity")
+)
+
+// writeByBlock writes len(p) bytes from p to the io.Writer in blocks of size blockSize.
+// It returns the number of bytes written from p (0 <= n <= len(p))
+// and any error encountered that caused the write to stop early.
+//
+// A pointer bytesInBlock to a counter needs to be supplied,
+// that is used to keep track how many bytes have been written to the writer already.
+// A pointer onlyNullBytesInBlock to a boolean needs to be supplied,
+// that is used to keep track whether the block so far only consists of null-bytes.
+// The callback onBlockWritten is called whenever a full block has been written to the writer
+// and is given as input the number of bytes that still need to be written.
+func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *uint32, onlyNullBytesInBlock *bool, onBlockWritten func(remaining int) error) (n int, err error) {
+	total := len(p)
+	nullBytes := make([]byte, blockSize)
+	for len(p) > 0 {
+		toWrite := int(blockSize - *bytesInBlock)
+		if toWrite > len(p) {
+			toWrite = len(p)
+		}
+		c, err := writer.Write(p[:toWrite])
+		*bytesInBlock += uint32(c)
+		*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
+		// Discard data written through a reslice
+		p = p[c:]
+		if err != nil {
+			return total - len(p), err
+		}
+		if *bytesInBlock == blockSize {
+			err = onBlockWritten(len(p))
+			if err != nil {
+				return total - len(p), err
+			}
+			*bytesInBlock = 0
+			*onlyNullBytesInBlock = true
+		}
+	}
+	return total, nil
+}
+
+// level is a hash.Hash that is used to aggregate the checksums produced by the level hierarchically beneath it.
+// It is used to represent any level-n hash, except for level-0.
+type level struct {
+	checksum              [Size]byte // aggregated checksum of this level
+	sumCount              uint32     // number of sums contained in this level so far
+	bytesInHasher         uint32     //  number of bytes written into hasher so far
+	onlyNullBytesInHasher bool       // whether the hasher only contains null-bytes so far
+	hasher                hash.Hash
+}
+
+// NewLevel returns a new hash.Hash computing any level-n hash, except level-0.
+func NewLevel() hash.Hash {
+	l := &level{}
+	l.Reset()
+	return l
+}
+
+// Add takes a position-embedded SHA-1 checksum and adds it to the level.
+func (l *level) Add(sha1sum []byte) {
+	var tmp uint
+	var carry bool
+	for i := Size - 1; i >= 0; i-- {
+		tmp = uint(sha1sum[i]) + uint(l.checksum[i])
+		if carry {
+			tmp++
+		}
+		carry = tmp > 255
+		l.checksum[i] = byte(tmp)
+	}
+}
+
+// IsFull returns whether the number of checksums added to this level reached its capacity.
+func (l *level) IsFull() bool {
+	return l.sumCount >= sumsPerLevel
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// Contrary to the specification from hash.Hash, this DOES return an error,
+// specifically ErrorHashFull if and only if IsFull() returns true.
+func (l *level) Write(p []byte) (n int, err error) {
+	if l.IsFull() {
+		return 0, ErrorHashFull
+	}
+	onBlockWritten := func(remaining int) error {
+		if !l.onlyNullBytesInHasher {
+			c, err := l.hasher.Write([]byte{byte(l.sumCount)})
+			l.bytesInHasher += uint32(c)
+			if err != nil {
+				return err
+			}
+			l.Add(l.hasher.Sum(nil))
+		}
+		l.sumCount++
+		l.hasher.Reset()
+		if remaining > 0 && l.IsFull() {
+			return ErrorHashFull
+		}
+		return nil
+	}
+	return writeByBlock(p, l.hasher, uint32(l.BlockSize()), &l.bytesInHasher, &l.onlyNullBytesInHasher, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (l *level) Sum(b []byte) []byte {
+	return append(b, l.checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (l *level) Reset() {
+	l.checksum = zeroSum // clear the current checksum
+	l.sumCount = 0
+	l.bytesInHasher = 0
+	l.onlyNullBytesInHasher = true
+	l.hasher = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (l *level) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (l *level) BlockSize() int {
+	return Size
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (l *level) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+4+1)
+	copy(b, l.checksum[:])
+	binary.BigEndian.PutUint32(b[Size:], l.sumCount)
+	binary.BigEndian.PutUint32(b[Size+4:], l.bytesInHasher)
+	if l.onlyNullBytesInHasher {
+		b[Size+4+4] = 1
+	}
+	encodedHasher, err := l.hasher.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedHasher...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (l *level) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+4+1 {
+		return ErrorInvalidEncoding
+	}
+	copy(l.checksum[:], b)
+	l.sumCount = binary.BigEndian.Uint32(b[Size:])
+	l.bytesInHasher = binary.BigEndian.Uint32(b[Size+4:])
+	switch b[Size+4+4] {
+	case 0:
+		l.onlyNullBytesInHasher = false
+	case 1:
+		l.onlyNullBytesInHasher = true
+	default:
+		return ErrorInvalidEncoding
+	}
+	err := l.hasher.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[Size+4+4+1:])
+	return err
+}
+
+// hidriveHash is the hash computing the actual checksum used by HiDrive by combining multiple level-hashes.
+type hidriveHash struct {
+	levels               []*level   // collection of level-hashes, one for each level starting at level-1
+	lastSumWritten       [Size]byte // the last checksum written to any of the levels
+	bytesInBlock         uint32     // bytes written into blockHash so far
+	onlyNullBytesInBlock bool       // whether the hasher only contains null-bytes so far
+	blockHash            hash.Hash
+}
+
+// New returns a new hash.Hash computing the HiDrive checksum.
+func New() hash.Hash {
+	h := &hidriveHash{}
+	h.Reset()
+	return h
+}
+
+// aggregateToLevel writes the checksum to the level at the given index
+// and if necessary propagates any changes to levels above.
+func (h *hidriveHash) aggregateToLevel(index int, sum []byte) {
+	for i := index; ; i++ {
+		if i >= len(h.levels) {
+			h.levels = append(h.levels, NewLevel().(*level))
+		}
+		_, err := h.levels[i].Write(sum)
+		copy(h.lastSumWritten[:], sum)
+		if err != nil {
+			panic(fmt.Errorf("level-hash should not have produced an error: %w", err))
+		}
+		if !h.levels[i].IsFull() {
+			break
+		}
+		sum = h.levels[i].Sum(nil)
+		h.levels[i].Reset()
+	}
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+func (h *hidriveHash) Write(p []byte) (n int, err error) {
+	onBlockWritten := func(remaining int) error {
+		var sum []byte
+		if h.onlyNullBytesInBlock {
+			sum = zeroSum[:]
+		} else {
+			sum = h.blockHash.Sum(nil)
+		}
+		h.blockHash.Reset()
+		h.aggregateToLevel(0, sum)
+		return nil
+	}
+	return writeByBlock(p, h.blockHash, uint32(BlockSize), &h.bytesInBlock, &h.onlyNullBytesInBlock, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (h *hidriveHash) Sum(b []byte) []byte {
+	// Save internal state.
+	state, err := h.MarshalBinary()
+	if err != nil {
+		panic(fmt.Errorf("saving the internal state should not have produced an error: %w", err))
+	}
+
+	if h.bytesInBlock > 0 {
+		// Fill remainder of block with null-bytes.
+		filler := make([]byte, h.BlockSize()-int(h.bytesInBlock))
+		_, err = h.Write(filler)
+		if err != nil {
+			panic(fmt.Errorf("filling with null-bytes should not have an error: %w", err))
+		}
+	}
+
+	checksum := zeroSum
+	for i := 0; i < len(h.levels); i++ {
+		level := h.levels[i]
+		if i < len(h.levels)-1 {
+			// Aggregate non-empty non-final levels.
+			if level.sumCount >= 1 {
+				h.aggregateToLevel(i+1, level.Sum(nil))
+				level.Reset()
+			}
+		} else {
+			// Determine sum of final level.
+			if level.sumCount > 1 {
+				copy(checksum[:], level.Sum(nil))
+			} else {
+				// This is needed, otherwise there is no way to return
+				// the non-position-embedded checksum.
+				checksum = h.lastSumWritten
+			}
+		}
+	}
+
+	// Restore internal state.
+	err = h.UnmarshalBinary(state)
+	if err != nil {
+		panic(fmt.Errorf("restoring the internal state should not have produced an error: %w", err))
+	}
+
+	return append(b, checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (h *hidriveHash) Reset() {
+	h.levels = nil
+	h.lastSumWritten = zeroSum // clear the last written checksum
+	h.bytesInBlock = 0
+	h.onlyNullBytesInBlock = true
+	h.blockHash = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (h *hidriveHash) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (h *hidriveHash) BlockSize() int {
+	return BlockSize
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (h *hidriveHash) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+1+8)
+	copy(b, h.lastSumWritten[:])
+	binary.BigEndian.PutUint32(b[Size:], h.bytesInBlock)
+	if h.onlyNullBytesInBlock {
+		b[Size+4] = 1
+	}
+
+	binary.BigEndian.PutUint64(b[Size+4+1:], uint64(len(h.levels)))
+	for _, level := range h.levels {
+		encodedLevel, err := level.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+		encodedLength := make([]byte, 8)
+		binary.BigEndian.PutUint64(encodedLength, uint64(len(encodedLevel)))
+		b = append(b, encodedLength...)
+		b = append(b, encodedLevel...)
+	}
+	encodedBlockHash, err := h.blockHash.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedBlockHash...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (h *hidriveHash) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+1+8 {
+		return ErrorInvalidEncoding
+	}
+	copy(h.lastSumWritten[:], b)
+	h.bytesInBlock = binary.BigEndian.Uint32(b[Size:])
+	switch b[Size+4] {
+	case 0:
+		h.onlyNullBytesInBlock = false
+	case 1:
+		h.onlyNullBytesInBlock = true
+	default:
+		return ErrorInvalidEncoding
+	}
+
+	amount := binary.BigEndian.Uint64(b[Size+4+1:])
+	h.levels = make([]*level, int(amount))
+	offset := Size + 4 + 1 + 8
+	for i := range h.levels {
+		length := int(binary.BigEndian.Uint64(b[offset:]))
+		offset += 8
+		h.levels[i] = NewLevel().(*level)
+		err := h.levels[i].UnmarshalBinary(b[offset : offset+length])
+		if err != nil {
+			return err
+		}
+		offset += length
+	}
+	err := h.blockHash.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[offset:])
+	return err
+}
+
+// Sum returns the HiDrive checksum of the data.
+func Sum(data []byte) [Size]byte {
+	h := New().(*hidriveHash)
+	_, _ = h.Write(data)
+	var result [Size]byte
+	copy(result[:], h.Sum(nil))
+	return result
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ hash.Hash                  = (*level)(nil)
+	_ encoding.BinaryMarshaler   = (*level)(nil)
+	_ encoding.BinaryUnmarshaler = (*level)(nil)
+	_ internal.LevelHash         = (*level)(nil)
+	_ hash.Hash                  = (*hidriveHash)(nil)
+	_ encoding.BinaryMarshaler   = (*hidriveHash)(nil)
+	_ encoding.BinaryUnmarshaler = (*hidriveHash)(nil)
+)

backend/hidrive/hidrivehash/hidrivehash_test.go

@@ -0,0 +1,395 @@
+package hidrivehash_test
+
+import (
+	"crypto/sha1"
+	"encoding"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash"
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+// helper functions to set up test-tables
+
+func sha1ArrayAsSlice(sum [sha1.Size]byte) []byte {
+	return sum[:]
+}
+
+func mustDecode(hexstring string) []byte {
+	result, err := hex.DecodeString(hexstring)
+	if err != nil {
+		panic(err)
+	}
+	return result
+}
+
+// ------------------------------------------------------------
+
+var testTableLevelPositionEmbedded = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{78, 188, 156, 219, 173, 54, 81, 55, 47, 220, 222, 207, 201, 21, 57, 252, 255, 239, 251, 186}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{68, 135, 96, 187, 38, 253, 14, 167, 186, 167, 188, 210, 91, 177, 185, 13, 208, 217, 94, 18}),
+		},
+		"documentation-v3.2rev27-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{75, 211, 153, 190, 125, 179, 67, 49, 60, 149, 98, 246, 142, 20, 11, 254, 159, 162, 129, 237}),
+			sha1ArrayAsSlice([20]byte{150, 2, 9, 153, 97, 153, 189, 104, 147, 14, 77, 203, 244, 243, 25, 212, 67, 48, 111, 107}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{144, 209, 246, 100, 177, 216, 171, 229, 83, 17, 92, 135, 68, 98, 76, 72, 217, 24, 99, 176}),
+			sha1ArrayAsSlice([20]byte{38, 211, 255, 254, 19, 114, 105, 77, 230, 31, 170, 83, 57, 85, 102, 29, 28, 72, 211, 27}),
+		},
+		"documentation-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{40, 34, 8, 238, 37, 5, 237, 184, 79, 105, 10, 167, 171, 254, 13, 229, 132, 112, 254, 8}),
+			sha1ArrayAsSlice([20]byte{39, 112, 26, 86, 190, 35, 100, 101, 28, 131, 122, 191, 254, 144, 239, 107, 253, 124, 104, 203}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{213, 157, 141, 227, 213, 178, 25, 111, 200, 145, 77, 164, 17, 247, 202, 167, 37, 46, 0, 124}),
+			sha1ArrayAsSlice([20]byte{253, 13, 168, 58, 147, 213, 125, 212, 229, 20, 200, 100, 16, 136, 186, 19, 34, 170, 105, 71}),
+		},
+		"documentation-example L1 (position-embedded)",
+	},
+}
+
+var testTableLevel = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		},
+		[][]byte{
+			mustDecode("44fe5ca6342568b4167bf990b64e404a3975e1c3"),
+			mustDecode("90d1f664b1d8abe553115c8744624c48d91863b0"),
+			mustDecode("26d3fffe1372694de61faa533955661d1c48d31b"),
+		},
+		"documentation-example L0",
+	},
+	{
+		[][]byte{
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+		},
+		[][]byte{
+			mustDecode("ad7b84f5b0ac2bb7792842fc65f9bcc1a0bd0274"),
+			mustDecode("d59d8de3d5b2196fc8914da411f7caa7252e007c"),
+			mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		},
+		"documentation-example L1",
+	},
+	{
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+			mustDecode("0000000000000000000000000000000000000000"),
+		},
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+		},
+		"mixed-with-empties",
+	},
+}
+
+var testTable = []struct {
+	data []byte
+	// pattern describes how to use data to construct the hash-input.
+	// For every entry n at even indices this repeats the data n times.
+	// For every entry m at odd indices this repeats a null-byte m times.
+	// The input-data is constructed by concatenating the results in order.
+	pattern []int64
+	out     []byte
+	name    string
+}{
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64},
+		mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		"documentation-example L0",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256},
+		mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+		"documentation-example L1",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256, 0, 64 * 128, 4096 * 128, 64*2 + 32},
+		mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		"documentation-example L2",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{316},
+		mustDecode("72370f9c18a2c20b31d71f3f4cee7a3cd2703737"),
+		"not-block-aligned",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{13, 4096 * 3, 4},
+		mustDecode("a6990b81791f0d2db750b38f046df321c975aa60"),
+		"not-block-aligned-with-null-bytes",
+	},
+	{
+		[]byte{},
+		[]int64{},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"empty",
+	},
+	{
+		[]byte{},
+		[]int64{0, 4096 * 256 * 256},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"null-bytes",
+	},
+}
+
+// ------------------------------------------------------------
+
+func TestLevelAdd(t *testing.T) {
+	for _, test := range testTableLevelPositionEmbedded {
+		l := hidrivehash.NewLevel().(internal.LevelHash)
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Add(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelWrite(t *testing.T) {
+	for _, test := range testTableLevel {
+		l := hidrivehash.NewLevel()
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Write(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelIsFull(t *testing.T) {
+	content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel()
+	for i := 0; i < 256; i++ {
+		assert.False(t, l.(internal.LevelHash).IsFull())
+		written, err := l.Write(content[:])
+		assert.Equal(t, len(content), written)
+		if !assert.NoError(t, err) {
+			t.FailNow()
+		}
+	}
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	written, err := l.Write(content[:])
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	assert.Equal(t, 0, written)
+	assert.ErrorIs(t, err, hidrivehash.ErrorHashFull)
+}
+
+func TestLevelReset(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	zeroHash := l.Sum(nil)
+	_, err := l.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, l.Sum(nil))
+		l.Reset()
+		assert.Equal(t, zeroHash, l.Sum(nil))
+	}
+}
+
+func TestLevelSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.Size())
+}
+
+func TestLevelBlockSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.BlockSize())
+}
+
+func TestLevelBinaryMarshaler(t *testing.T) {
+	content := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	l.Write(content[:10])
+	encoded, err := l.MarshalBinary()
+	if assert.NoError(t, err) {
+		d := hidrivehash.NewLevel().(internal.LevelHash)
+		err = d.UnmarshalBinary(encoded)
+		if assert.NoError(t, err) {
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+			l.Write(content[10:])
+			d.Write(content[10:])
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+		}
+	}
+}
+
+func TestLevelInvalidEncoding(t *testing.T) {
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	err := l.UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+// ------------------------------------------------------------
+
+type infiniteReader struct {
+	source []byte
+	offset int
+}
+
+func (m *infiniteReader) Read(b []byte) (int, error) {
+	count := copy(b, m.source[m.offset:])
+	m.offset += count
+	m.offset %= len(m.source)
+	return count, nil
+}
+
+func writeInChunks(writer io.Writer, chunkSize int64, data []byte, pattern []int64) error {
+	readers := make([]io.Reader, len(pattern))
+	nullBytes := [4096]byte{}
+	for i, n := range pattern {
+		if i%2 == 0 {
+			readers[i] = io.LimitReader(&infiniteReader{data, 0}, n*int64(len(data)))
+		} else {
+			readers[i] = io.LimitReader(&infiniteReader{nullBytes[:], 0}, n)
+		}
+	}
+	reader := io.MultiReader(readers...)
+	for {
+		_, err := io.CopyN(writer, reader, chunkSize)
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return err
+		}
+	}
+}
+
+func TestWrite(t *testing.T) {
+	for _, test := range testTable {
+		t.Run(test.name, func(t *testing.T) {
+			h := hidrivehash.New()
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern)
+			if assert.NoError(t, err) {
+				normalSum := h.Sum(nil)
+				assert.Equal(t, test.out, normalSum)
+				// Test if different block-sizes produce differing results.
+				for _, blockSize := range []int64{397, 512, 4091, 8192, 10000} {
+					t.Run(fmt.Sprintf("block-size %v", blockSize), func(t *testing.T) {
+						h := hidrivehash.New()
+						err := writeInChunks(h, blockSize, test.data, test.pattern)
+						if assert.NoError(t, err) {
+							assert.Equal(t, normalSum, h.Sum(nil))
+						}
+					})
+				}
+			}
+		})
+	}
+}
+
+func TestReset(t *testing.T) {
+	h := hidrivehash.New()
+	zeroHash := h.Sum(nil)
+	_, err := h.Write([]byte{1})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, h.Sum(nil))
+		h.Reset()
+		assert.Equal(t, zeroHash, h.Sum(nil))
+	}
+}
+
+func TestSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 20, h.Size())
+}
+
+func TestBlockSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 4096, h.BlockSize())
+}
+
+func TestBinaryMarshaler(t *testing.T) {
+	for _, test := range testTable {
+		h := hidrivehash.New()
+		d := hidrivehash.New()
+		half := len(test.pattern) / 2
+		t.Run(test.name, func(t *testing.T) {
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[:half])
+			assert.NoError(t, err)
+			encoded, err := h.(encoding.BinaryMarshaler).MarshalBinary()
+			if assert.NoError(t, err) {
+				err = d.(encoding.BinaryUnmarshaler).UnmarshalBinary(encoded)
+				if assert.NoError(t, err) {
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+					err = writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					err = writeInChunks(d, int64(d.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+				}
+			}
+		})
+	}
+}
+
+func TestInvalidEncoding(t *testing.T) {
+	h := hidrivehash.New()
+	err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+func TestSum(t *testing.T) {
+	assert.Equal(t, [hidrivehash.Size]byte{}, hidrivehash.Sum([]byte{}))
+	content := []byte{1}
+	h := hidrivehash.New()
+	h.Write(content)
+	sum := hidrivehash.Sum(content)
+	assert.Equal(t, h.Sum(nil), sum[:])
+}

backend/hidrive/hidrivehash/internal/internal.go

@@ -0,0 +1,18 @@
+// Package internal provides utilities for HiDrive.
+package internal
+
+import (
+	"encoding"
+	"hash"
+)
+
+// LevelHash is an internal interface for level-hashes.
+type LevelHash interface {
+	encoding.BinaryMarshaler
+	encoding.BinaryUnmarshaler
+	hash.Hash
+	// Add takes a position-embedded checksum and adds it to the level.
+	Add(sum []byte)
+	// IsFull returns whether the number of checksums added to this level reached its capacity.
+	IsFull() bool
+}

backend/http/http.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -35,11 +34,11 @@ var (
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "http",
-		Description: "http Connection",
+		Description: "HTTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "url",
-			Help:     "URL of http host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
+			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
 			Required: true,
 		}, {
 			Name: "headers",
@@ -305,7 +304,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat(ctx)
+	err := o.head(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -317,15 +316,6 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-// parse s into an int64, on failure return def
-func parseInt64(s string, def int64) int64 {
-	n, e := strconv.ParseInt(s, 10, 64)
-	if e != nil {
-		return def
-	}
-	return n
-}
-
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -500,12 +490,12 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.stat(ctx); err {
+				switch err := file.head(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
 					// ...found a directory not a file
-					add(fs.NewDir(remote, timeUnset))
+					add(fs.NewDir(remote, time.Time{}))
 				default:
 					fs.Debugf(remote, "skipping because of error: %v", err)
 				}
@@ -517,7 +507,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		name = strings.TrimRight(name, "/")
 		remote := path.Join(dir, name)
 		if isDir {
-			add(fs.NewDir(remote, timeUnset))
+			add(fs.NewDir(remote, time.Time{}))
 		} else {
 			in <- remote
 		}
@@ -579,8 +569,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// stat updates the info field in the Object
-func (o *Object) stat(ctx context.Context) error {
+// head sends a HEAD request to update info fields in the Object
+func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -601,13 +591,19 @@ func (o *Object) stat(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
+	return o.decodeMetadata(ctx, res)
+}
+
+// decodeMetadata updates info fields in the Object according to HTTP response headers
+func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	o.size = rest.ParseSizeFromHeaders(res.Header)
+
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -653,6 +649,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
+	if err = o.decodeMetadata(ctx, res); err != nil {
+		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
+	}
 	return res.Body, nil
 }
 

backend/http/http_internal_test.go

@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -33,20 +33,21 @@ var (
 	lineEndSize = 1
 )
 
-// prepareServer the test server and return a function to tidy it up afterwards
-func prepareServer(t *testing.T) (configmap.Simple, func()) {
+// prepareServer prepares the test server and shuts it down automatically
+// when the test completes.
+func prepareServer(t *testing.T) configmap.Simple {
 	// file server for test/files
 	fileServer := http.FileServer(http.Dir(filesPath))
 
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := ioutil.ReadDir(filesPath)
+	fileList, err := os.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -78,20 +79,21 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 		"url":     ts.URL,
 		"headers": strings.Join(headers, ","),
 	}
+	t.Cleanup(ts.Close)
 
-	// return a function to tidy up
-	return m, ts.Close
+	return m
 }
 
-// prepare the test server and return a function to tidy it up afterwards
-func prepare(t *testing.T) (fs.Fs, func()) {
-	m, tidy := prepareServer(t)
+// prepare prepares the test server and shuts it down automatically
+// when the test completes.
+func prepare(t *testing.T) fs.Fs {
+	m := prepareServer(t)
 
 	// Instantiate it
 	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
-	return f, tidy
+	return f
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
@@ -134,22 +136,19 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 }
 
 func TestListRoot(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 	testListRoot(t, f, false)
 }
 
 func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
+	f := prepare(t)
 	f.(*Fs).opt.NoSlash = true
-	defer tidy()
 
 	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
@@ -166,8 +165,7 @@ func TestListSubDir(t *testing.T) {
 }
 
 func TestNewObject(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -194,36 +192,69 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	m := prepareServer(t)
 
-	o, err := f.NewObject(context.Background(), "four/under four.txt")
-	require.NoError(t, err)
+	for _, head := range []bool{false, true} {
+		if !head {
+			m.Set("no_head", "true")
+		}
+		f, err := NewFs(context.Background(), remoteName, "", m)
+		require.NoError(t, err)
 
-	// Test normal read
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	if lineEndSize == 2 {
-		assert.Equal(t, "beetroot\r\n", string(data))
-	} else {
-		assert.Equal(t, "beetroot\n", string(data))
+		for _, rangeRead := range []bool{false, true} {
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			if !head {
+				// Test mod time is still indeterminate
+				tObj := o.ModTime(context.Background())
+				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
+
+				// Test file size is still indeterminate
+				assert.Equal(t, int64(-1), o.Size())
+			}
+
+			var data []byte
+			if !rangeRead {
+				// Test normal read
+				fd, err := o.Open(context.Background())
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				if lineEndSize == 2 {
+					assert.Equal(t, "beetroot\r\n", string(data))
+				} else {
+					assert.Equal(t, "beetroot\n", string(data))
+				}
+			} else {
+				// Test with range request
+				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				assert.Equal(t, "eetro", string(data))
+			}
+
+			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+			require.NoError(t, err)
+			tFile := fi.ModTime()
+
+			// Test the time is always correct on the object after file open
+			tObj := o.ModTime(context.Background())
+			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+			if !rangeRead {
+				// Test the file size
+				assert.Equal(t, int64(len(data)), o.Size())
+			}
+		}
 	}
-
-	// Test with range request
-	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-	require.NoError(t, err)
-	data, err = ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, "eetro", string(data))
 }
 
 func TestMimeType(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -234,8 +265,7 @@ func TestMimeType(t *testing.T) {
 }
 
 func TestIsAFileRoot(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
@@ -244,8 +274,7 @@ func TestIsAFileRoot(t *testing.T) {
 }
 
 func TestIsAFileSubDir(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)

backend/hubic/auth.go

@@ -1,62 +0,0 @@
-package hubic
-
-import (
-	"context"
-	"net/http"
-	"time"
-
-	"github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/fs"
-)
-
-// auth is an authenticator for swift
-type auth struct {
-	f *Fs
-}
-
-// newAuth creates a swift authenticator
-func newAuth(f *Fs) *auth {
-	return &auth{
-		f: f,
-	}
-}
-
-// Request constructs an http.Request for authentication
-//
-// returns nil for not needed
-func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Request, err error) {
-	const retries = 10
-	for try := 1; try <= retries; try++ {
-		err = a.f.getCredentials(context.TODO())
-		if err == nil {
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-		fs.Debugf(a.f, "retrying auth request %d/%d: %v", try, retries, err)
-	}
-	return nil, err
-}
-
-// Response parses the result of an http request
-func (a *auth) Response(ctx context.Context, resp *http.Response) error {
-	return nil
-}
-
-// The public storage URL - set Internal to true to read
-// internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string { // nolint
-	return a.f.credentials.Endpoint
-}
-
-// The access token
-func (a *auth) Token() string {
-	return a.f.credentials.Token
-}
-
-// The CDN url if available
-func (a *auth) CdnUrl() string { // nolint
-	return ""
-}
-
-// Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)

backend/hubic/hubic.go

@@ -1,200 +0,0 @@
-// Package hubic provides an interface to the Hubic object storage
-// system.
-package hubic
-
-// This uses the normal swift mechanism to update the credentials and
-// ignores the expires field returned by the Hubic API.  This may need
-// to be revisited after some actual experience.
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"time"
-
-	swiftLib "github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/backend/swift"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/oauthutil"
-	"golang.org/x/oauth2"
-)
-
-const (
-	rcloneClientID              = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
-	rcloneEncryptedClientSecret = "leZKCcqy9movLhDWLVXX8cSLp_FzoiAPeEJOIOMRw1A5RuC4iLEPDYPWVF46adC_MVonnLdVEOTHVstfBOZ_lY4WNp8CK_YWlpRZ9diT5YI"
-)
-
-// Globals
-var (
-	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
-		Scopes: []string{
-			"credentials.r", // Read OpenStack credentials
-		},
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://api.hubic.com/oauth/auth/",
-			TokenURL: "https://api.hubic.com/oauth/token/",
-		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
-	}
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "hubic",
-		Description: "Hubic",
-		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
-			})
-		},
-		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
-	})
-}
-
-// credentials is the JSON returned from the Hubic API to read the
-// OpenStack credentials
-type credentials struct {
-	Token    string `json:"token"`    // OpenStack token
-	Endpoint string `json:"endpoint"` // OpenStack endpoint
-	Expires  string `json:"expires"`  // Expires date - e.g. "2015-11-09T14:24:56+01:00"
-}
-
-// Fs represents a remote hubic
-type Fs struct {
-	fs.Fs                    // wrapped Fs
-	features    *fs.Features // optional features
-	client      *http.Client // client for oauth api
-	credentials credentials  // returned from the Hubic API
-	expires     time.Time    // time credentials expire
-}
-
-// Object describes a swift object
-type Object struct {
-	*swift.Object
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.Object.String()
-}
-
-// ------------------------------------------------------------
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	if f.Fs == nil {
-		return "Hubic"
-	}
-	return fmt.Sprintf("Hubic %s", f.Fs.String())
-}
-
-// getCredentials reads the OpenStack Credentials using the Hubic API
-//
-// The credentials are read into the Fs
-func (f *Fs) getCredentials(ctx context.Context) (err error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "https://api.hubic.com/1.0/account/credentials", nil)
-	if err != nil {
-		return err
-	}
-	resp, err := f.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer fs.CheckClose(resp.Body, &err)
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
-		return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
-	}
-	decoder := json.NewDecoder(resp.Body)
-	var result credentials
-	err = decoder.Decode(&result)
-	if err != nil {
-		return err
-	}
-	// fs.Debugf(f, "Got credentials %+v", result)
-	if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
-		return errors.New("couldn't read token, result and expired from credentials")
-	}
-	f.credentials = result
-	expires, err := time.Parse(time.RFC3339, result.Expires)
-	if err != nil {
-		return err
-	}
-	f.expires = expires
-	fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
-	return nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	client, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to configure Hubic: %w", err)
-	}
-
-	f := &Fs{
-		client: client,
-	}
-
-	// Make the swift Connection
-	ci := fs.GetConfig(ctx)
-	c := &swiftLib.Connection{
-		Auth:           newAuth(f),
-		ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * ci.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(ctx),
-	}
-	err = c.Authenticate(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error authenticating swift connection: %w", err)
-	}
-
-	// Parse config into swift.Options struct
-	opt := new(swift.Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Make inner swift Fs from the connection
-	swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true)
-	if err != nil && err != fs.ErrorIsFile {
-		return nil, err
-	}
-	f.Fs = swiftFs
-	f.features = f.Fs.Features().Wrap(f)
-	return f, err
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// UnWrap returns the Fs that this Fs is wrapping
-func (f *Fs) UnWrap() fs.Fs {
-	return f.Fs
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs        = (*Fs)(nil)
-	_ fs.UnWrapper = (*Fs)(nil)
-)

backend/hubic/hubic_test.go

@@ -1,19 +0,0 @@
-// Test Hubic filesystem interface
-package hubic_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/hubic"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:          "TestHubic:",
-		NilObject:           (*hubic.Object)(nil),
-		SkipFsCheckWrap:     true,
-		SkipObjectCheckWrap: true,
-	})
-}

backend/internetarchive/internetarchive.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -37,12 +38,108 @@ func init() {
 		Name:        "internetarchive",
 		Description: "Internet Archive",
 		NewFs:       NewFs,
+
+		MetadataInfo: &fs.MetadataInfo{
+			System: map[string]fs.MetadataHelp{
+				"name": {
+					Help:     "Full file path, without the bucket part",
+					Type:     "filename",
+					Example:  "backend/internetarchive/internetarchive.go",
+					ReadOnly: true,
+				},
+				"source": {
+					Help:     "The source of the file",
+					Type:     "string",
+					Example:  "original",
+					ReadOnly: true,
+				},
+				"mtime": {
+					Help:     "Time of last modification, managed by Rclone",
+					Type:     "RFC 3339",
+					Example:  "2006-01-02T15:04:05.999999999Z",
+					ReadOnly: true,
+				},
+				"size": {
+					Help:     "File size in bytes",
+					Type:     "decimal number",
+					Example:  "123456",
+					ReadOnly: true,
+				},
+				"md5": {
+					Help:     "MD5 hash calculated by Internet Archive",
+					Type:     "string",
+					Example:  "01234567012345670123456701234567",
+					ReadOnly: true,
+				},
+				"crc32": {
+					Help:     "CRC32 calculated by Internet Archive",
+					Type:     "string",
+					Example:  "01234567",
+					ReadOnly: true,
+				},
+				"sha1": {
+					Help:     "SHA1 hash calculated by Internet Archive",
+					Type:     "string",
+					Example:  "0123456701234567012345670123456701234567",
+					ReadOnly: true,
+				},
+				"format": {
+					Help:     "Name of format identified by Internet Archive",
+					Type:     "string",
+					Example:  "Comma-Separated Values",
+					ReadOnly: true,
+				},
+				"old_version": {
+					Help:     "Whether the file was replaced and moved by keep-old-version flag",
+					Type:     "boolean",
+					Example:  "true",
+					ReadOnly: true,
+				},
+				"viruscheck": {
+					Help:     "The last time viruscheck process was run for the file (?)",
+					Type:     "unixtime",
+					Example:  "1654191352",
+					ReadOnly: true,
+				},
+				"summation": {
+					Help:     "Check https://forum.rclone.org/t/31922 for how it is used",
+					Type:     "string",
+					Example:  "md5",
+					ReadOnly: true,
+				},
+
+				"rclone-ia-mtime": {
+					Help:    "Time of last modification, managed by Internet Archive",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z",
+				},
+				"rclone-mtime": {
+					Help:    "Time of last modification, managed by Rclone",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z",
+				},
+				"rclone-update-track": {
+					Help:    "Random value used by Rclone for tracking changes inside Internet Archive",
+					Type:    "string",
+					Example: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				},
+			},
+			Help: `Metadata fields provided by Internet Archive.
+If there are multiple values for a key, only the first one is returned.
+This is a limitation of Rclone, that supports one value per one key.
+
+Owner is able to add custom keys. Metadata feature grabs all the keys including them.
+`,
+		},
+
 		Options: []fs.Option{{
-			Name: "access_key_id",
-			Help: "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
+			Name:      "access_key_id",
+			Help:      "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
+			Sensitive: true,
 		}, {
-			Name: "secret_access_key",
-			Help: "IAS3 Secret Key (password).\n\nLeave blank for anonymous access.",
+			Name:      "secret_access_key",
+			Help:      "IAS3 Secret Key (password).\n\nLeave blank for anonymous access.",
+			Sensitive: true,
 		}, {
 			// their official client (https://github.com/jjjake/internetarchive) hardcodes following the two
 			Name:     "endpoint",
@@ -89,6 +186,14 @@ Only enable if you need to be guaranteed to be reflected after write operations.
 // maximum size of an item. this is constant across all items
 const iaItemMaxSize int64 = 1099511627776
 
+// metadata keys that are not writeable
+var roMetadataKey = map[string]interface{}{
+	// do not add mtime here, it's a documented exception
+	"name": nil, "source": nil, "size": nil, "md5": nil,
+	"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
+	"viruscheck": nil, "summation": nil,
+}
+
 // Options defines the configuration for this backend
 type Options struct {
 	AccessKeyID     string               `config:"access_key_id"`
@@ -121,26 +226,37 @@ type Object struct {
 	md5     string    // md5 hash of the file presented by the server
 	sha1    string    // sha1 hash of the file presented by the server
 	crc32   string    // crc32 of the file presented by the server
+	rawData json.RawMessage
 }
 
-// IAFile reprensents a subset of object in MetadataResponse.Files
+// IAFile represents a subset of object in MetadataResponse.Files
 type IAFile struct {
 	Name string `json:"name"`
 	// Source     string `json:"source"`
 	Mtime       string          `json:"mtime"`
 	RcloneMtime json.RawMessage `json:"rclone-mtime"`
+	UpdateTrack json.RawMessage `json:"rclone-update-track"`
 	Size        string          `json:"size"`
 	Md5         string          `json:"md5"`
 	Crc32       string          `json:"crc32"`
 	Sha1        string          `json:"sha1"`
+	Summation   string          `json:"summation"`
+
+	rawData json.RawMessage
 }
 
-// MetadataResponse reprensents subset of the JSON object returned by (frontend)/metadata/
+// MetadataResponse represents subset of the JSON object returned by (frontend)/metadata/
 type MetadataResponse struct {
 	Files    []IAFile `json:"files"`
 	ItemSize int64    `json:"item_size"`
 }
 
+// MetadataResponseRaw is the form of MetadataResponse to deal with metadata
+type MetadataResponseRaw struct {
+	Files    []json.RawMessage `json:"files"`
+	ItemSize int64             `json:"item_size"`
+}
+
 // ModMetadataResponse represents response for amending metadata
 type ModMetadataResponse struct {
 	// https://archive.org/services/docs/api/md-write.html#example
@@ -224,7 +340,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
-		BucketBased: true,
+		BucketBased:   true,
+		ReadMetadata:  true,
+		WriteMetadata: true,
+		UserMetadata:  true,
 	}).Fill(ctx, f)
 
 	f.srv = rest.NewClient(fshttp.NewClient(ctx))
@@ -294,7 +413,7 @@ func (o *Object) Storable() bool {
 	return true
 }
 
-// SetModTime is not supported
+// SetModTime sets modTime on a particular file
 func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	bucket, reqDir := o.split()
 	if bucket == "" {
@@ -305,18 +424,17 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	}
 
 	// https://archive.org/services/docs/api/md-write.html
-	var patch = []interface{}{
+	// the following code might be useful for modifying metadata of an uploaded file
+	patch := []map[string]string{
 		// we should drop it first to clear all rclone-provided mtimes
-		struct {
-			Op   string `json:"op"`
-			Path string `json:"path"`
-		}{"remove", "/rclone-mtime"},
-		struct {
-			Op    string `json:"op"`
-			Path  string `json:"path"`
-			Value string `json:"value"`
-		}{"add", "/rclone-mtime", t.Format(time.RFC3339Nano)},
-	}
+		{
+			"op":   "remove",
+			"path": "/rclone-mtime",
+		}, {
+			"op":    "add",
+			"path":  "/rclone-mtime",
+			"value": t.Format(time.RFC3339Nano),
+		}}
 	res, err := json.Marshal(patch)
 	if err != nil {
 		return err
@@ -456,14 +574,14 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return "", err
 	}
 	bucket, bucketPath := f.split(remote)
-	return path.Join(f.opt.FrontEndpoint, "/download/", bucket, bucketPath), nil
+	return path.Join(f.opt.FrontEndpoint, "/download/", bucket, quotePath(bucketPath)), nil
 }
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -483,6 +601,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		return nil, fs.ErrorCantCopy
 	}
 
+	updateTracker := random.String(32)
 	headers := map[string]string{
 		"x-archive-auto-make-bucket": "1",
 		"x-archive-queue-derive":     "0",
@@ -494,7 +613,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		"x-archive-filemeta-crc32":   srcObj.crc32,
 		"x-archive-filemeta-size":    fmt.Sprint(srcObj.size),
 		// add this too for sure
-		"x-archive-filemeta-rclone-mtime": srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-mtime":        srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-update-track": updateTracker,
 	}
 
 	// make a PUT request at (IAS3)/:item/:path without body
@@ -515,7 +635,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 
 	// we can't update/find metadata here as IA will also
 	// queue server-side copy as well as upload/delete.
-	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), f.getHashes(ctx, src), srcObj.size)
+	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), updateTracker, srcObj.size)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -642,7 +762,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	// make a GET request to (frontend)/download/:item/:path
 	opts := rest.Opts{
 		Method:  "GET",
-		Path:    path.Join("/download/", o.fs.root, o.fs.opt.Enc.FromStandardPath(o.remote)),
+		Path:    path.Join("/download/", o.fs.root, quotePath(o.fs.opt.Enc.FromStandardPath(o.remote))),
 		Options: optionsFixed,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -660,12 +780,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	bucket, bucketPath := o.split()
 	modTime := src.ModTime(ctx)
 	size := src.Size()
+	updateTracker := random.String(32)
 
 	// Set the mtime in the metadata
 	// internetarchive backend builds at header level as IAS3 has extension outside X-Amz-
 	headers := map[string]string{
 		// https://github.com/jjjake/internetarchive/blob/2456376533251df9d05e0a14d796ec1ced4959f5/internetarchive/iarequest.py#L158
-		"x-amz-filemeta-rclone-mtime": modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-mtime":        modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
 		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
@@ -679,6 +801,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		headers["Content-Length"] = fmt.Sprintf("%d", size)
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
+	var mdata fs.Metadata
+	mdata, err = fs.GetMetadataOptions(ctx, src, options)
+	if err == nil && mdata != nil {
+		for mk, mv := range mdata {
+			mk = strings.ToLower(mk)
+			if strings.HasPrefix(mk, "rclone-") {
+				fs.LogPrintf(fs.LogLevelWarning, o, "reserved metadata key %s is about to set", mk)
+			} else if _, ok := roMetadataKey[mk]; ok {
+				fs.LogPrintf(fs.LogLevelWarning, o, "setting or modifying read-only key %s is requested, skipping", mk)
+				continue
+			} else if mk == "mtime" {
+				// redirect to make it work
+				mk = "rclone-mtime"
+			}
+			headers[fmt.Sprintf("x-amz-filemeta-%s", mk)] = mv
+		}
+	}
 
 	// read the md5sum if available
 	var md5sumHex string
@@ -712,7 +851,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)
 	var newObj *Object
 	if err == nil {
-		newObj, err = o.fs.waitFileUpload(ctx, o.remote, o.fs.getHashes(ctx, src), size)
+		newObj, err = o.fs.waitFileUpload(ctx, o.remote, updateTracker, size)
 	} else {
 		newObj = &Object{}
 	}
@@ -756,6 +895,34 @@ func (o *Object) String() string {
 	return o.remote
 }
 
+// Metadata returns all file metadata provided by Internet Archive
+func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error) {
+	if o.rawData == nil {
+		return nil, nil
+	}
+	raw := make(map[string]json.RawMessage)
+	err = json.Unmarshal(o.rawData, &raw)
+	if err != nil {
+		// fatal: json parsing failed
+		return
+	}
+	for k, v := range raw {
+		items, err := listOrString(v)
+		if len(items) == 0 || err != nil {
+			// skip: an entry failed to parse
+			continue
+		}
+		m.Set(k, items[0])
+	}
+	// move the old mtime to an another key
+	if v, ok := m["mtime"]; ok {
+		m["rclone-ia-mtime"] = v
+	}
+	// overwrite with a correct mtime
+	m["mtime"] = o.modTime.Format(time.RFC3339Nano)
+	return
+}
+
 func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	if resp != nil {
 		for _, e := range retryErrorCodes {
@@ -782,19 +949,7 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-func (f *Fs) getHashes(ctx context.Context, src fs.ObjectInfo) map[hash.Type]string {
-	hashMap := map[hash.Type]string{}
-	for _, ty := range f.Hashes().Array() {
-		sum, err := src.Hash(ctx, ty)
-		if err != nil || sum == "" {
-			continue
-		}
-		hashMap[ty] = sum
-	}
-	return hashMap
-}
-
-func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result MetadataResponse, err error) {
+func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result *MetadataResponse, err error) {
 	var resp *http.Response
 	// make a GET request to (frontend)/metadata/:item/
 	opts := rest.Opts{
@@ -802,12 +957,15 @@ func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result Metadat
 		Path:   path.Join("/metadata/", bucket),
 	}
 
+	var temp MetadataResponseRaw
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.front.CallJSON(ctx, &opts, nil, &result)
+		resp, err = f.front.CallJSON(ctx, &opts, nil, &temp)
 		return f.shouldRetry(resp, err)
 	})
-
-	return result, err
+	if err != nil {
+		return
+	}
+	return temp.unraw()
 }
 
 // list up all files/directories without any filters
@@ -852,7 +1010,7 @@ func (f *Fs) listAllUnconstrained(ctx context.Context, bucket string) (entries f
 	return entries, nil
 }
 
-func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[hash.Type]string, newSize int64) (ret *Object, err error) {
+func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSize int64) (ret *Object, err error) {
 	bucket, bucketPath := f.split(reqPath)
 
 	ret = &Object{
@@ -869,6 +1027,10 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			ret2, ok := ret2.(*Object)
 			if ok {
 				ret = ret2
+				ret.crc32 = ""
+				ret.md5 = ""
+				ret.sha1 = ""
+				ret.size = -1
 			}
 		}
 		return ret, nil
@@ -881,9 +1043,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 	go func() {
 		isFirstTime := true
 		existed := false
-		oldMtime := ""
-		oldCrc32 := ""
-		unreliablePassCount := 0
 		for {
 			if !isFirstTime {
 				// depending on the queue, it takes time
@@ -908,10 +1067,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			if isFirstTime {
 				isFirstTime = false
 				existed = iaFile != nil
-				if iaFile != nil {
-					oldMtime = iaFile.Mtime
-					oldCrc32 = iaFile.Crc32
-				}
 			}
 			if iaFile == nil {
 				continue
@@ -925,38 +1080,20 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 				return
 			}
 
-			hashMatched := true
-			for tt, sum := range newHashes {
-				if tt == hash.MD5 && !hash.Equals(iaFile.Md5, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.SHA1 && !hash.Equals(iaFile.Sha1, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.CRC32 && !hash.Equals(iaFile.Crc32, sum) {
-					hashMatched = false
+			fileTrackers, _ := listOrString(iaFile.UpdateTrack)
+			trackerMatch := false
+			for _, v := range fileTrackers {
+				if v == tracker {
+					trackerMatch = true
 					break
 				}
 			}
-			if !hashMatched {
+			if !trackerMatch {
 				continue
 			}
 			if !compareSize(parseSize(iaFile.Size), newSize) {
 				continue
 			}
-			if hash.Equals(oldCrc32, iaFile.Crc32) && unreliablePassCount < 60 {
-				// the following two are based on a sort of "bad" assumption;
-				// what if the file is updated immediately, before polling?
-				// by limiting hits of these tests, avoid infinite loop
-				unreliablePassCount++
-				continue
-			}
-			if hash.Equals(iaFile.Mtime, oldMtime) && unreliablePassCount < 60 {
-				unreliablePassCount++
-				continue
-			}
 
 			// voila!
 			retC <- struct {
@@ -1017,15 +1154,21 @@ func (f *Fs) waitDelete(ctx context.Context, bucket, bucketPath string) (err err
 }
 
 func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object {
-	return &Object{
+	ret := &Object{
 		fs:      f,
 		remote:  remote,
 		modTime: mtime,
 		size:    size,
-		md5:     file.Md5,
-		crc32:   file.Crc32,
-		sha1:    file.Sha1,
+		rawData: file.rawData,
 	}
+	// hashes from _files.xml (where summation != "") is different from one in other files
+	// https://forum.rclone.org/t/internet-archive-md5-tag-in-id-files-xml-interpreted-incorrectly/31922
+	if file.Summation == "" {
+		ret.md5 = file.Md5
+		ret.crc32 = file.Crc32
+		ret.sha1 = file.Sha1
+	}
+	return ret
 }
 
 func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {
@@ -1036,20 +1179,24 @@ func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {
 	return makeValidObject(f, trimPathPrefix(path.Join(bucket, file.Name), f.root, f.opt.Enc), file, mtimeTime, size)
 }
 
-func (file IAFile) parseMtime() (mtime time.Time) {
-	// method 1: use metadata added by rclone
-	var rmArray []string
+func listOrString(jm json.RawMessage) (rmArray []string, err error) {
 	// rclone-metadata can be an array or string
 	// try to deserialize it as array first
-	err := json.Unmarshal(file.RcloneMtime, &rmArray)
+	err = json.Unmarshal(jm, &rmArray)
 	if err != nil {
 		// if not, it's a string
 		dst := new(string)
-		err = json.Unmarshal(file.RcloneMtime, dst)
+		err = json.Unmarshal(jm, dst)
 		if err == nil {
 			rmArray = []string{*dst}
 		}
 	}
+	return
+}
+
+func (file IAFile) parseMtime() (mtime time.Time) {
+	// method 1: use metadata added by rclone
+	rmArray, err := listOrString(file.RcloneMtime)
 	// let's take the first value we can deserialize
 	for _, value := range rmArray {
 		mtime, err = time.Parse(time.RFC3339Nano, value)
@@ -1068,6 +1215,23 @@ func (file IAFile) parseMtime() (mtime time.Time) {
 	return mtime
 }
 
+func (mrr *MetadataResponseRaw) unraw() (_ *MetadataResponse, err error) {
+	var files []IAFile
+	for _, raw := range mrr.Files {
+		var parsed IAFile
+		err = json.Unmarshal(raw, &parsed)
+		if err != nil {
+			return nil, err
+		}
+		parsed.rawData = raw
+		files = append(files, parsed)
+	}
+	return &MetadataResponse{
+		Files:    files,
+		ItemSize: mrr.ItemSize,
+	}, nil
+}
+
 func compareSize(a, b int64) bool {
 	if a < 0 || b < 0 {
 		// we won't compare if any of them is not known
@@ -1111,7 +1275,7 @@ func trimPathPrefix(s, prefix string, enc encoder.MultiEncoder) string {
 	return enc.ToStandardPath(strings.TrimPrefix(s, prefix+"/"))
 }
 
-// mimicks urllib.parse.quote() on Python; exclude / from url.PathEscape
+// mimics urllib.parse.quote() on Python; exclude / from url.PathEscape
 func quotePath(s string) string {
 	seg := strings.Split(s, "/")
 	newValues := []string{}
@@ -1129,4 +1293,5 @@ var (
 	_ fs.PublicLinker = &Fs{}
 	_ fs.Abouter      = &Fs{}
 	_ fs.Object       = &Object{}
+	_ fs.Metadataer   = &Object{}
 )

backend/jottacloud/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Jottacloud API.
 package api
 
 import (

backend/jottacloud/jottacloud.go

@@ -1,3 +1,4 @@
+// Package jottacloud provides an interface to the Jottacloud storage system.
 package jottacloud
 
 import (
@@ -11,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -46,9 +46,9 @@ const (
 	decayConstant      = 2 // bigger for slower decay, exponential
 	defaultDevice      = "Jotta"
 	defaultMountpoint  = "Archive"
-	rootURL            = "https://jfs.jottacloud.com/jfs/"
+	jfsURL             = "https://jfs.jottacloud.com/jfs/"
 	apiURL             = "https://api.jottacloud.com/"
-	baseURL            = "https://www.jottacloud.com/"
+	wwwURL             = "https://www.jottacloud.com/"
 	cachePrefix        = "rclone-jcmd5-"
 	configDevice       = "device"
 	configMountpoint   = "mountpoint"
@@ -74,6 +74,10 @@ const (
 	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
 	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
 	tele2CloudClientID = "desktop"
+
+	onlimeCloudTokenURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/token"
+	onlimeCloudAuthURL  = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/auth"
+	onlimeCloudClientID = "desktop"
 )
 
 // Register with Fs
@@ -84,7 +88,7 @@ func init() {
 		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config:      Config,
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
@@ -119,7 +123,7 @@ func init() {
 			Default: (encoder.Display |
 				encoder.EncodeWin | // :?"*<>|
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 
@@ -127,7 +131,7 @@ func init() {
 func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
 	switch config.State {
 	case "":
-		return fs.ConfigChooseFixed("auth_type_done", "config_type", `Authentication type.`, []fs.OptionExample{{
+		return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{
 			Value: "standard",
 			Help:  "Standard authentication.\nUse this if you're a normal Jottacloud user.",
 		}, {
@@ -139,13 +143,16 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 		}, {
 			Value: "tele2",
 			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
+		}, {
+			Value: "onlime",
+			Help:  "Onlime Cloud authentication.\nUse this if you are using Onlime Cloud.",
 		}})
 	case "auth_type_done":
 		// Jump to next state according to config chosen
 		return fs.ConfigGoto(config.Result)
 	case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
 		m.Set("configVersion", fmt.Sprint(configVersion))
-		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\n\nGenerate here: https://www.jottacloud.com/web/secure")
+		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure")
 	case "standard_token":
 		loginToken := config.Result
 		m.Set(configClientID, defaultClientID)
@@ -191,7 +198,7 @@ machines.`)
 		m.Set("auth_code", "")
 		return fs.ConfigGoto("legacy_do_auth")
 	case "legacy_auth_code":
-		authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
+		authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
 		m.Set("auth_code", authCode)
 		return fs.ConfigGoto("legacy_do_auth")
 	case "legacy_do_auth":
@@ -261,8 +268,27 @@ machines.`)
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
 		})
+	case "onlime": // onlime cloud config
+		m.Set("configVersion", fmt.Sprint(configVersion))
+		m.Set(configClientID, onlimeCloudClientID)
+		m.Set(configTokenURL, onlimeCloudTokenURL)
+		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  onlimeCloudAuthURL,
+					TokenURL: onlimeCloudTokenURL,
+				},
+				ClientID:    onlimeCloudClientID,
+				Scopes:      []string{"openid", "jotta-default", "offline_access"},
+				RedirectURL: oauthutil.RedirectLocalhostURL,
+			},
+		})
 	case "choose_device":
-		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
+		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
+Choosing no, the default, will let you access the storage used for the archive
+section of the official Jottacloud client. If you instead want to access the
+sync or the backup section, for example, you must choose yes.`)
+
 	case "choose_device_query":
 		if config.Result != "true" {
 			m.Set(configDevice, "")
@@ -273,43 +299,139 @@ machines.`)
 		if err != nil {
 			return nil, err
 		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
 		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
 
 		cust, err := getCustomerInfo(ctx, apiSrv)
 		if err != nil {
 			return nil, err
 		}
-		m.Set(configUsername, cust.Username)
 
-		acc, err := getDriveInfo(ctx, srv, cust.Username)
+		acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
 		if err != nil {
 			return nil, err
 		}
-		return fs.ConfigChoose("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) {
-			return acc.Devices[i].Name, ""
+
+		deviceNames := make([]string, len(acc.Devices))
+		for i, dev := range acc.Devices {
+			if i > 0 && dev.Name == defaultDevice {
+				// Insert the special Jotta device as first entry, making it the default choice.
+				copy(deviceNames[1:i+1], deviceNames[0:i])
+				deviceNames[0] = dev.Name
+			} else {
+				deviceNames[i] = dev.Name
+			}
+		}
+
+		help := fmt.Sprintf(`The device to use. In standard setup the built-in %s device is used,
+which contains predefined mountpoints for archive, sync etc. All other devices
+are treated as backup devices by the official Jottacloud client. You may create
+a new by entering a unique name.`, defaultDevice)
+		return fs.ConfigChoose("choose_device_result", "config_device", help, len(deviceNames), func(i int) (string, string) {
+			return deviceNames[i], ""
 		})
 	case "choose_device_result":
 		device := config.Result
-		m.Set(configDevice, device)
 
 		oAuthClient, _, err := getOAuthClient(ctx, name, m)
 		if err != nil {
 			return nil, err
 		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
 
-		username, _ := m.Get(configUsername)
-		dev, err := getDeviceInfo(ctx, srv, path.Join(username, device))
+		cust, err := getCustomerInfo(ctx, apiSrv)
 		if err != nil {
 			return nil, err
 		}
-		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) {
+
+		acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
+		if err != nil {
+			return nil, err
+		}
+		isNew := true
+		for _, dev := range acc.Devices {
+			if strings.EqualFold(dev.Name, device) { // If device name exists with different casing we prefer the existing (not sure if and how the api handles the opposite)
+				device = dev.Name // Prefer same casing as existing, e.g. if user entered "jotta" we use the standard casing "Jotta" instead
+				isNew = false
+				break
+			}
+		}
+		var dev *api.JottaDevice
+		if isNew {
+			fs.Debugf(nil, "Creating new device: %s", device)
+			dev, err = createDevice(ctx, jfsSrv, path.Join(cust.Username, device))
+			if err != nil {
+				return nil, err
+			}
+		}
+		m.Set(configDevice, device)
+
+		if !isNew {
+			dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		var help string
+		if device == defaultDevice {
+			// With built-in Jotta device the mountpoint choice is exclusive,
+			// we do not want to risk any problems by creating new mountpoints on it.
+			help = fmt.Sprintf(`The mountpoint to use on the built-in device %s.
+The standard setup is to use the %s mountpoint. Most other mountpoints
+have very limited support in rclone and should generally be avoided.`, defaultDevice, defaultMountpoint)
+			return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
+				return dev.MountPoints[i].Name, ""
+			})
+		}
+		help = fmt.Sprintf(`The mountpoint to use on the non-standard device %s.
+You may create a new by entering a unique name.`, device)
+		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
 			return dev.MountPoints[i].Name, ""
 		})
 	case "choose_device_mountpoint":
 		mountpoint := config.Result
+
+		oAuthClient, _, err := getOAuthClient(ctx, name, m)
+		if err != nil {
+			return nil, err
+		}
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		cust, err := getCustomerInfo(ctx, apiSrv)
+		if err != nil {
+			return nil, err
+		}
+
+		device, _ := m.Get(configDevice)
+
+		dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
+		if err != nil {
+			return nil, err
+		}
+		isNew := true
+		for _, mnt := range dev.MountPoints {
+			if strings.EqualFold(mnt.Name, mountpoint) {
+				mountpoint = mnt.Name
+				isNew = false
+				break
+			}
+		}
+
+		if isNew {
+			if device == defaultDevice {
+				return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err)
+			}
+			fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint)
+			_, err := createMountPoint(ctx, jfsSrv, path.Join(cust.Username, device, mountpoint))
+			if err != nil {
+				return nil, err
+			}
+		}
 		m.Set(configMountpoint, mountpoint)
+
 		return fs.ConfigGoto("end")
 	case "end":
 		// All the config flows end up here in case we need to carry on with something
@@ -332,16 +454,17 @@ type Options struct {
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name         string
-	root         string
-	user         string
-	opt          Options
-	features     *fs.Features
-	endpointURL  string
-	srv          *rest.Client
-	apiSrv       *rest.Client
-	pacer        *fs.Pacer
-	tokenRenewer *oauthutil.Renew // renew the token on expiry
+	name             string
+	root             string
+	user             string
+	opt              Options
+	features         *fs.Features
+	fileEndpoint     string
+	allocateEndpoint string
+	jfsSrv           *rest.Client
+	apiSrv           *rest.Client
+	pacer            *fs.Pacer
+	tokenRenewer     *oauthutil.Renew // renew the token on expiry
 }
 
 // Object describes a jottacloud object
@@ -588,15 +711,47 @@ func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *ap
 	return info, nil
 }
 
-// setEndpointURL generates the API endpoint URL
-func (f *Fs) setEndpointURL() {
+// createDevice makes a device
+func createDevice(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       urlPathEscape(path),
+		Parameters: url.Values{},
+	}
+
+	opts.Parameters.Set("type", "WORKSTATION")
+
+	_, err = srv.CallXML(ctx, &opts, nil, &info)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create device: %w", err)
+	}
+	return info, nil
+}
+
+// createMountPoint makes a mount point
+func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info *api.JottaMountPoint, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   urlPathEscape(path),
+	}
+
+	_, err = srv.CallXML(ctx, &opts, nil, &info)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create mountpoint: %w", err)
+	}
+	return info, nil
+}
+
+// setEndpoints generates the API endpoints
+func (f *Fs) setEndpoints() {
 	if f.opt.Device == "" {
 		f.opt.Device = defaultDevice
 	}
 	if f.opt.Mountpoint == "" {
 		f.opt.Mountpoint = defaultMountpoint
 	}
-	f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
+	f.fileEndpoint = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
+	f.allocateEndpoint = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint)
 }
 
 // readMetaDataForPath reads the metadata from the path
@@ -608,7 +763,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
 	var result api.JottaFile
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -649,17 +804,34 @@ func errorHandler(resp *http.Response) error {
 
 // Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
 func urlPathEscape(in string) string {
-	return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
+	return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
 }
 
 // filePathRaw returns an unescaped file path (f.root, file)
-func (f *Fs) filePathRaw(file string) string {
-	return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
+// Optionally made absolute by prefixing with "/", typically required when used
+// as request parameter instead of the path (which is relative to some root url).
+func (f *Fs) filePathRaw(file string, absolute bool) string {
+	prefix := ""
+	if absolute {
+		prefix = "/"
+	}
+	return path.Join(prefix, f.fileEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
 // filePath returns an escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
-	return urlPathEscape(f.filePathRaw(file))
+	return urlPathEscape(f.filePathRaw(file, false))
+}
+
+// allocatePathRaw returns an unescaped allocate file path (f.root, file)
+// Optionally made absolute by prefixing with "/", typically required when used
+// as request parameter instead of the path (which is relative to some root url).
+func (f *Fs) allocatePathRaw(file string, absolute bool) string {
+	prefix := ""
+	if absolute {
+		prefix = "/"
+	}
+	return path.Join(prefix, f.allocateEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
 // Jottacloud requires the grant_type 'refresh_token' string
@@ -671,7 +843,7 @@ func (f *Fs) filePath(file string) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
+		refreshBody, err := io.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -681,7 +853,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -692,12 +864,12 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	if ok {
 		ver, err = strconv.Atoi(version)
 		if err != nil {
-			return nil, nil, errors.New("Failed to parse config version")
+			return nil, nil, errors.New("failed to parse config version")
 		}
 		ok = (ver == configVersion) || (ver == legacyConfigVersion)
 	}
 	if !ok {
-		return nil, nil, errors.New("Outdated config - please reconfigure this backend")
+		return nil, nil, errors.New("outdated config - please reconfigure this backend")
 	}
 
 	baseClient := fshttp.NewClient(ctx)
@@ -743,7 +915,7 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	// Create OAuth Client
 	oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to configure Jottacloud oauth client: %w", err)
+		return nil, nil, fmt.Errorf("failed to configure Jottacloud oauth client: %w", err)
 	}
 	return oAuthClient, ts, nil
 }
@@ -769,7 +941,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:   name,
 		root:   root,
 		opt:    *opt,
-		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
+		jfsSrv: rest.NewClient(oAuthClient).SetRoot(jfsURL),
 		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
 		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
@@ -779,7 +951,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ReadMimeType:            true,
 		WriteMimeType:           false,
 	}).Fill(ctx, f)
-	f.srv.SetErrorHandler(errorHandler)
+	f.jfsSrv.SetErrorHandler(errorHandler)
 	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
 		f.features.ListR = nil
 	}
@@ -798,7 +970,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 	f.user = cust.Username
-	f.setEndpointURL()
+	f.setEndpoints()
 
 	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
@@ -864,7 +1036,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
 	opts.Parameters.Set("mkDir", "true")
 
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &jf)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -893,7 +1065,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var resp *http.Response
 	var result api.JottaFolder
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -932,7 +1104,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
-func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
+func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
 
 	type stats struct {
 		Folders int `xml:"folders"`
@@ -968,8 +1140,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 		})
 	}
 
+	// liststream paths are /mountpoint/root/path
+	// so the returned paths should have /mountpoint/root/ trimmed
+	// as the caller is expecting path.
+	pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
 	trimPathPrefix := func(p string) string {
-		p = strings.TrimPrefix(p, trimPrefix)
+		p = strings.TrimPrefix(p, pathPrefix)
 		p = strings.TrimPrefix(p, "/")
 		return p
 	}
@@ -1026,7 +1202,7 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 
 	if expected.Folders != actual.Folders ||
 		expected.Files != actual.Files {
-		return fmt.Errorf("Invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
+		return fmt.Errorf("invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
 	}
 	return nil
 }
@@ -1047,16 +1223,12 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
+		resp, err = f.jfsSrv.Call(ctx, &opts)
 		if err != nil {
 			return shouldRetry(ctx, resp, err)
 		}
 
-		// liststream paths are /mountpoint/root/path
-		// so the returned paths should have /mountpoint/root/ trimmed
-		// as the caller is expecting path.
-		trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
-		err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
+		err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
 			if d.Remote() == dir {
 				return nil
 			}
@@ -1097,13 +1269,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if f.opt.Device != "Jotta" {
-		return nil, errors.New("upload not supported for devices other than Jotta")
-	}
 	o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size())
 	return o, o.Update(ctx, in, src, options...)
 }
@@ -1113,10 +1282,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	// chop off trailing / if it exists
-	if strings.HasSuffix(dirPath, "/") {
-		dirPath = dirPath[:len(dirPath)-1]
-	}
-	parent := path.Dir(dirPath)
+	parent := path.Dir(strings.TrimSuffix(dirPath, "/"))
 	if parent == "." {
 		parent = ""
 	}
@@ -1164,7 +1330,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
+		resp, err = f.jfsSrv.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1217,7 +1383,7 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time,
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -1238,11 +1404,11 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, dest))))
+	opts.Parameters.Set(method, f.filePathRaw(dest, true))
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1253,9 +1419,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1273,7 +1439,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
-	// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
+	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
 	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
 		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
 		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
@@ -1289,9 +1455,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1351,7 +1517,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
+	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.fileEndpoint, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
 	if err != nil {
 		return fmt.Errorf("couldn't move directory: %w", err)
@@ -1376,7 +1542,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	var result api.JottaFile
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -1402,19 +1568,19 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return "", errors.New("couldn't create public link - no uri received")
 	}
 	if result.PublicSharePath != "" {
-		webLink := joinPath(baseURL, result.PublicSharePath)
+		webLink := joinPath(wwwURL, result.PublicSharePath)
 		fs.Debugf(nil, "Web link: %s", webLink)
 	} else {
 		fs.Debugf(nil, "No web link received")
 	}
-	directLink := joinPath(baseURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
+	directLink := joinPath(wwwURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
 	fs.Debugf(nil, "Direct link: %s", directLink)
 	return directLink, nil
 }
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
-	info, err := getDriveInfo(ctx, f.srv, f.user)
+	info, err := getDriveInfo(ctx, f.jfsSrv, f.user)
 	if err != nil {
 		return nil, err
 	}
@@ -1617,7 +1783,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	opts.Parameters.Set("mode", "bin")
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
+		resp, err = o.fs.jfsSrv.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1644,7 +1810,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = ioutil.TempFile("", cachePrefix)
+		tempFile, err = os.CreateTemp("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1672,7 +1838,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = ioutil.ReadAll(teeReader)
+		inData, err = io.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1685,7 +1851,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
@@ -1694,12 +1860,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err == nil {
 			// if the object exists delete it
 			err = o.remove(ctx, true)
-			if err != nil {
+			if err != nil && err != fs.ErrorObjectNotFound {
+				// if delete failed then report that, unless it was because the file did not exist after all
 				return fmt.Errorf("failed to remove old object: %w", err)
 			}
-		}
-		// if the object does not exist we can just continue but if the error is something different we should report that
-		if err != fs.ErrorObjectNotFound {
+		} else if err != fs.ErrorObjectNotFound {
+			// if the object does not exist we can just continue but if the error is something different we should report that
 			return err
 		}
 	}
@@ -1738,7 +1904,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Created:  fileDate,
 		Modified: fileDate,
 		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
+		Path:     o.fs.allocatePathRaw(o.remote, true),
 	}
 
 	// send it
@@ -1769,7 +1935,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		_, err = io.CopyN(io.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}
@@ -1786,7 +1952,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.md5 = result.Md5
 		o.modTime = time.Unix(result.Modified/1000, 0)
 	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
+		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
 		return o.readMetaData(ctx, true)
 	}
 
@@ -1807,10 +1973,17 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 		opts.Parameters.Set("dl", "true")
 	}
 
-	return o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
+	err := o.fs.pacer.Call(func() (bool, error) {
+		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
 		return shouldRetry(ctx, resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// attempting to hard delete will fail if path does not exist, but standard delete will succeed
+		if apiErr.StatusCode == http.StatusNotFound {
+			return fs.ErrorObjectNotFound
+		}
+	}
+	return err
 }
 
 // Remove an object

backend/koofr/koofr.go

@@ -1,3 +1,4 @@
+// Package koofr provides an interface to the Koofr storage system.
 package koofr
 
 import (
@@ -60,9 +61,10 @@ func init() {
 			Default:  true,
 			Advanced: true,
 		}, {
-			Name:     "user",
-			Help:     "Your user name.",
-			Required: true,
+			Name:      "user",
+			Help:      "Your user name.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:       "password",
 			Help:       "Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password).",
@@ -351,9 +353,9 @@ func NewFsFromOptions(ctx context.Context, name, root string, opt *Options) (ff
 	}
 	if f.mountID == "" {
 		if opt.MountID == "" {
-			return nil, errors.New("Failed to find primary mount")
+			return nil, errors.New("failed to find primary mount")
 		}
-		return nil, errors.New("Failed to find mount " + opt.MountID)
+		return nil, errors.New("failed to find mount " + opt.MountID)
 	}
 	rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root))
 	if err == nil && rootFile.Type != "dir" {
@@ -375,7 +377,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	for i, file := range files {
 		remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name))
 		if file.Type == "dir" {
-			entries[i] = fs.NewDir(remote, time.Unix(0, 0))
+			entries[i] = fs.NewDir(remote, time.Time{})
 		} else {
 			entries[i] = &Object{
 				fs:     f,
@@ -667,7 +669,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	//
 	// https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F
 	//
-	// I am not sure about meaning of "path" parameter; in my expriments
+	// I am not sure about meaning of "path" parameter; in my experiments
 	// it is always "%2F", and omitting it or putting any other value
 	// results in 404.
 	//

backend/local/about_windows.go

@@ -17,8 +17,12 @@ var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSp
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var available, total, free int64
+	root, e := syscall.UTF16PtrFromString(f.root)
+	if e != nil {
+		return nil, fmt.Errorf("failed to read disk usage: %w", e)
+	}
 	_, _, e1 := getFreeDiskSpace.Call(
-		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))),
+		uintptr(unsafe.Pointer(root)),
 		uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user
 		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
 		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes

backend/local/local.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -22,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -42,9 +42,22 @@ func init() {
 		Description: "Local Disk",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
+		MetadataInfo: &fs.MetadataInfo{
+			System: systemMetadataInfo,
+			Help: `Depending on which OS is in use the local backend may return only some
+of the system metadata. Setting system metadata is supported on all
+OSes but setting user metadata is only supported on linux, freebsd,
+netbsd, macOS and Solaris. It is **not** supported on Windows yet
+([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)).
+
+User metadata is stored as extended attributes (which may not be
+supported by all file systems) under the "user.*" prefix.
+`,
+		},
 		Options: []fs.Option{{
 			Name:     "nounc",
 			Help:     "Disable UNC (long path names) conversion on Windows.",
+			Default:  false,
 			Advanced: runtime.GOOS != "windows",
 			Examples: []fs.OptionExample{{
 				Value: "true",
@@ -110,8 +123,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy -
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
@@ -221,15 +234,16 @@ type Options struct {
 
 // Fs represents a local filesystem rooted at root
 type Fs struct {
-	name        string              // the name of the remote
-	root        string              // The root directory (OS path)
-	opt         Options             // parsed config options
-	features    *fs.Features        // optional features
-	dev         uint64              // device number of root node
-	precisionOk sync.Once           // Whether we need to read the precision
-	precision   time.Duration       // precision of local filesystem
-	warnedMu    sync.Mutex          // used for locking access to 'warned'.
-	warned      map[string]struct{} // whether we have warned about this string
+	name           string              // the name of the remote
+	root           string              // The root directory (OS path)
+	opt            Options             // parsed config options
+	features       *fs.Features        // optional features
+	dev            uint64              // device number of root node
+	precisionOk    sync.Once           // Whether we need to read the precision
+	precision      time.Duration       // precision of local filesystem
+	warnedMu       sync.Mutex          // used for locking access to 'warned'.
+	warned         map[string]struct{} // whether we have warned about this string
+	xattrSupported int32               // whether xattrs are supported (atomic access)
 
 	// do os.Lstat or os.Stat
 	lstat        func(name string) (os.FileInfo, error)
@@ -252,7 +266,10 @@ type Object struct {
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+var (
+	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
+)
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -273,12 +290,20 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		dev:    devUnset,
 		lstat:  os.Lstat,
 	}
+	if xattrSupported {
+		f.xattrSupported = 1
+	}
 	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
 		IsLocal:                 true,
 		SlowHash:                true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:             true,
+		PartialUploads:          true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -289,7 +314,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
+	// Check to see if this is a .rclonelink if not found
+	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
+	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
+		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
+	}
 	if err == nil && f.isRegular(fi.Mode()) {
+		// Handle the odd case, that a symlink was specified by name without the link suffix
+		if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+			return nil, errLinksNeedsSuffix
+		}
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -423,6 +457,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx)
+
 	fsDirPath := f.localPath(dir)
 	_, err = os.Stat(fsDirPath)
 	if err != nil {
@@ -473,7 +509,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 						continue
 					}
 					if fierr != nil {
-						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
+						// Don't report errors on any file names that are excluded
+						if useFilter {
+							newRemote := f.cleanRemote(dir, name)
+							if !filter.IncludeRemote(newRemote) {
+								continue
+							}
+						}
+						fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
@@ -494,6 +537,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
+				// Quietly skip errors on excluded files and directories
+				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
@@ -518,6 +565,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
+				// Don't include non directory if not included
+				// we leave directory filtering to the layer above
+				if useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err
@@ -610,7 +662,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := ioutil.TempFile("", "rclone")
+	fd, err := os.CreateTemp("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -679,9 +731,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -903,7 +955,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", fmt.Errorf("hash: failed to open: %w", err)
 		}
 		var hashes map[hash.Type]string
-		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
+		hashes, err = hash.StreamTypes(readers.NewContextReader(ctx, in), hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
 			return "", fmt.Errorf("hash: failed to read: %w", err)
@@ -937,17 +989,22 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 	return o.modTime
 }
 
+// Set the atime and ltime of the object
+func (o *Object) setTimes(atime, mtime time.Time) (err error) {
+	if o.translatedLink {
+		err = lChtimes(o.path, atime, mtime)
+	} else {
+		err = os.Chtimes(o.path, atime, mtime)
+	}
+	return err
+}
+
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if o.fs.opt.NoSetModTime {
 		return nil
 	}
-	var err error
-	if o.translatedLink {
-		err = lChtimes(o.path, modTime, modTime)
-	} else {
-		err = os.Chtimes(o.path, modTime, modTime)
-	}
+	err := o.setTimes(modTime, modTime)
 	if err != nil {
 		return err
 	}
@@ -1032,7 +1089,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read
@@ -1222,6 +1279,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
+	// Fetch and set metadata if --metadata is in use
+	meta, err := fs.GetMetadataOptions(ctx, src, options)
+	if err != nil {
+		return fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	err = o.writeMetadata(meta)
+	if err != nil {
+		return fmt.Errorf("failed to set metadata: %w", err)
+	}
+
 	// ReRead info now that we have finished
 	return o.lstat()
 }
@@ -1320,31 +1387,56 @@ func (o *Object) Remove(ctx context.Context) error {
 	return remove(o.path)
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	metadata, err = o.getXattr()
+	if err != nil {
+		return nil, err
+	}
+	err = o.readMetadataFromFile(&metadata)
+	if err != nil {
+		return nil, err
+	}
+	return metadata, nil
+}
+
+// Write the metadata on the object
+func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
+	err = o.setXattr(metadata)
+	if err != nil {
+		return err
+	}
+	err = o.writeMetadataToFile(metadata)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS == "windows" {
-		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
+	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
+		if !filepath.IsAbs(s) {
 			s2, err := filepath.Abs(s)
 			if err == nil {
 				s = s2
 			}
+		} else {
+			s = filepath.Clean(s)
 		}
+	}
+	if runtime.GOOS == "windows" {
 		s = filepath.ToSlash(s)
 		vol := filepath.VolumeName(s)
 		s = vol + enc.FromStandardPath(s[len(vol):])
 		s = filepath.FromSlash(s)
-
 		if !noUNC {
 			// Convert to UNC
 			s = file.UNCPath(s)
 		}
 		return s
 	}
-	if !filepath.IsAbs(s) {
-		s2, err := filepath.Abs(s)
-		if err == nil {
-			s = s2
-		}
-	}
 	s = enc.FromStandardPath(s)
 	return s
 }
@@ -1359,4 +1451,5 @@ var (
 	_ fs.Commander      = &Fs{}
 	_ fs.OpenWriterAter = &Fs{}
 	_ fs.Object         = &Object{}
+	_ fs.Metadataer     = &Object{}
 )

backend/local/local_internal_test.go

@@ -3,17 +3,23 @@ package local
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
+	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
+	"sort"
 	"testing"
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
@@ -29,7 +35,6 @@ func TestMain(m *testing.M) {
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
@@ -74,7 +79,6 @@ func TestUpdatingCheck(t *testing.T) {
 func TestSymlink(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	f := r.Flocal.(*Fs)
 	dir := f.root
 
@@ -143,10 +147,24 @@ func TestSymlink(t *testing.T) {
 	_, err = r.Flocal.NewObject(ctx, "symlink2.txt")
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
+	// Check that NewFs works with the suffixed version and --links
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.Equal(t, dir, f2.(*Fs).root)
+
+	// Check that NewFs doesn't see the non suffixed version with --links
+	f2, err = NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, errLinksNeedsSuffix, err)
+	require.Nil(t, f2)
+
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
+	contents, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -154,7 +172,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
+	contents, err = io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
@@ -173,7 +191,6 @@ func TestSymlinkError(t *testing.T) {
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -188,7 +205,7 @@ func TestHashOnUpdate(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
 
-	// Reupload it with diferent contents but same size and timestamp
+	// Reupload it with different contents but same size and timestamp
 	var b = bytes.NewBufferString("CONTENT")
 	src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
 	err = o.Update(ctx, b, src)
@@ -204,7 +221,6 @@ func TestHashOnUpdate(t *testing.T) {
 func TestHashOnDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -229,3 +245,313 @@ func TestHashOnDelete(t *testing.T) {
 	_, err = o.Hash(ctx, hash.MD5)
 	require.Error(t, err)
 }
+
+func TestMetadata(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	const filePath = "metafile.txt"
+	when := time.Now()
+	const dayLength = len("2001-01-01")
+	whenRFC := when.Format(time.RFC3339Nano)
+	r.WriteFile(filePath, "metadata file contents", when)
+	f := r.Flocal.(*Fs)
+
+	// Get the object
+	obj, err := f.NewObject(ctx, filePath)
+	require.NoError(t, err)
+	o := obj.(*Object)
+
+	features := f.Features()
+
+	var hasXID, hasAtime, hasBtime bool
+	switch runtime.GOOS {
+	case "darwin", "freebsd", "netbsd", "linux":
+		hasXID, hasAtime, hasBtime = true, true, true
+	case "openbsd", "solaris":
+		hasXID, hasAtime = true, true
+	case "windows":
+		hasAtime, hasBtime = true, true
+	case "plan9", "js":
+		// nada
+	default:
+		t.Errorf("No test cases for OS %q", runtime.GOOS)
+	}
+
+	assert.True(t, features.ReadMetadata)
+	assert.True(t, features.WriteMetadata)
+	assert.Equal(t, xattrSupported, features.UserMetadata)
+
+	t.Run("Xattr", func(t *testing.T) {
+		if !xattrSupported {
+			t.Skip()
+		}
+		m, err := o.getXattr()
+		require.NoError(t, err)
+		assert.Nil(t, m)
+
+		inM := fs.Metadata{
+			"potato":  "chips",
+			"cabbage": "soup",
+		}
+		err = o.setXattr(inM)
+		require.NoError(t, err)
+
+		m, err = o.getXattr()
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+		assert.Equal(t, inM, m)
+	})
+
+	checkTime := func(m fs.Metadata, key string, when time.Time) {
+		mt, ok := o.parseMetadataTime(m, key)
+		assert.True(t, ok)
+		dt := mt.Sub(when)
+		precision := time.Second
+		assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v", key, dt, precision))
+	}
+
+	checkInt := func(m fs.Metadata, key string, base int) int {
+		value, ok := o.parseMetadataInt(m, key, base)
+		assert.True(t, ok)
+		return value
+	}
+	t.Run("Read", func(t *testing.T) {
+		m, err := o.Metadata(ctx)
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+
+		// All OSes have these
+		checkInt(m, "mode", 8)
+		checkTime(m, "mtime", when)
+
+		assert.Equal(t, len(whenRFC), len(m["mtime"]))
+		assert.Equal(t, whenRFC[:dayLength], m["mtime"][:dayLength])
+
+		if hasAtime {
+			checkTime(m, "atime", when)
+		}
+		if hasBtime {
+			checkTime(m, "btime", when)
+		}
+		if hasXID {
+			checkInt(m, "uid", 10)
+			checkInt(m, "gid", 10)
+		}
+	})
+
+	t.Run("Write", func(t *testing.T) {
+		newAtimeString := "2011-12-13T14:15:16.999999999Z"
+		newAtime := fstest.Time(newAtimeString)
+		newMtimeString := "2011-12-12T14:15:16.999999999Z"
+		newMtime := fstest.Time(newMtimeString)
+		newBtimeString := "2011-12-11T14:15:16.999999999Z"
+		newBtime := fstest.Time(newBtimeString)
+		newM := fs.Metadata{
+			"mtime": newMtimeString,
+			"atime": newAtimeString,
+			"btime": newBtimeString,
+			// Can't test uid, gid without being root
+			"mode":   "0767",
+			"potato": "wedges",
+		}
+		err := o.writeMetadata(newM)
+		require.NoError(t, err)
+
+		m, err := o.Metadata(ctx)
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+
+		mode := checkInt(m, "mode", 8)
+		if runtime.GOOS != "windows" {
+			assert.Equal(t, 0767, mode&0777, fmt.Sprintf("mode wrong - expecting 0767 got 0%o", mode&0777))
+		}
+
+		checkTime(m, "mtime", newMtime)
+		if hasAtime {
+			checkTime(m, "atime", newAtime)
+		}
+		if haveSetBTime {
+			checkTime(m, "btime", newBtime)
+		}
+		if xattrSupported {
+			assert.Equal(t, "wedges", m["potato"])
+		}
+	})
+
+}
+
+func TestFilter(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	when := time.Now()
+	r.WriteFile("included", "included file", when)
+	r.WriteFile("excluded", "excluded file", when)
+	f := r.Flocal.(*Fs)
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included"))
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[excluded included]", fmt.Sprint(entries))
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[included]", fmt.Sprint(entries))
+}
+
+func testFilterSymlink(t *testing.T, copyLinks bool) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
+	r.WriteFile("included.file", "included file", when)
+	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
+	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
+	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
+	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
+
+	defer func() {
+		// Reset -L/-l mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Lstat
+	}()
+	if copyLinks {
+		// Set fs into "-L" mode
+		f.opt.FollowSymlinks = true
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Stat
+	} else {
+		// Set fs into "-l" mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = true
+		f.lstat = os.Lstat
+	}
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Reset global error count
+	accounting.Stats(ctx).ResetErrors()
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included.file"))
+	require.NoError(t, fi.AddRule("+ included.dir/**"))
+	if copyLinks {
+		require.NoError(t, fi.AddRule("+ included.file.link"))
+		require.NoError(t, fi.AddRule("+ included.dir.link/**"))
+	} else {
+		require.NoError(t, fi.AddRule("+ included.file.link.rclonelink"))
+		require.NoError(t, fi.AddRule("+ included.dir.link.rclonelink"))
+	}
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+
+	if copyLinks {
+		// Check 1 global errors one for each dangling symlink
+		assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+	} else {
+		// Check 0 global errors as dangling symlink copied properly
+		assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+	}
+	accounting.Stats(ctx).ResetErrors()
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[dangling.link.rclonelink included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Check listing through a symlink still works
+	entries, err = f.List(ctx, "included.dir")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
+}
+
+func TestFilterSymlinkCopyLinks(t *testing.T) {
+	testFilterSymlink(t, true)
+}
+
+func TestFilterSymlinkLinks(t *testing.T) {
+	testFilterSymlink(t, false)
+}
+
+func TestCopySymlink(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file and a symlink to it
+	r.WriteFile("src/file.txt", "hello world", when)
+	require.NoError(t, os.Symlink("file.txt", filepath.Join(r.LocalName, "src", "link.txt")))
+	defer func() {
+		// Reset -L/-l mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Lstat
+	}()
+
+	// Set fs into "-l/--links" mode
+	f.opt.FollowSymlinks = false
+	f.opt.TranslateSymlinks = true
+	f.lstat = os.Lstat
+
+	// Create dst
+	require.NoError(t, f.Mkdir(ctx, "dst"))
+
+	// Do copy from src into dst
+	src, err := f.NewObject(ctx, "src/link.txt.rclonelink")
+	require.NoError(t, err)
+	require.NotNil(t, src)
+	dst, err := operations.Copy(ctx, f, nil, "dst/link.txt.rclonelink", src)
+	require.NoError(t, err)
+	require.NotNil(t, dst)
+
+	// Test that we made a symlink and it has the right contents
+	dstPath := filepath.Join(r.LocalName, "dst", "link.txt")
+	linkContents, err := os.Readlink(dstPath)
+	require.NoError(t, err)
+	assert.Equal(t, "file.txt", linkContents)
+}

backend/local/local_test.go

@@ -11,7 +11,8 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "",
-		NilObject:  (*local.Object)(nil),
+		RemoteName:  "",
+		NilObject:   (*local.Object)(nil),
+		QuickTestOK: true,
 	})
 }

backend/local/metadata.go

@@ -0,0 +1,138 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+const metadataTimeFormat = time.RFC3339Nano
+
+// system metadata keys which this backend owns
+//
+// not all values supported on all OSes
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+	"mode": {
+		Help:    "File type and mode",
+		Type:    "octal, unix style",
+		Example: "0100664",
+	},
+	"uid": {
+		Help:    "User ID of owner",
+		Type:    "decimal number",
+		Example: "500",
+	},
+	"gid": {
+		Help:    "Group ID of owner",
+		Type:    "decimal number",
+		Example: "500",
+	},
+	"rdev": {
+		Help:    "Device ID (if special file)",
+		Type:    "hexadecimal",
+		Example: "1abc",
+	},
+	"atime": {
+		Help:    "Time of last access",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+	"mtime": {
+		Help:    "Time of last modification",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+	"btime": {
+		Help:    "Time of file birth (creation)",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+}
+
+// parse a time string from metadata with key
+func (o *Object) parseMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
+	value, ok := m[key]
+	if ok {
+		var err error
+		t, err = time.Parse(metadataTimeFormat, value)
+		if err != nil {
+			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
+			ok = false
+		}
+	}
+	return t, ok
+}
+
+// parse am int from metadata with key and base
+func (o *Object) parseMetadataInt(m fs.Metadata, key string, base int) (result int, ok bool) {
+	value, ok := m[key]
+	if ok {
+		var err error
+		result64, err := strconv.ParseInt(value, base, 64)
+		if err != nil {
+			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
+			ok = false
+		}
+		result = int(result64)
+	}
+	return result, ok
+}
+
+// Write the metadata into the file
+//
+// It isn't possible to set the ctime and btime under Unix
+func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
+	var err error
+	atime, atimeOK := o.parseMetadataTime(m, "atime")
+	mtime, mtimeOK := o.parseMetadataTime(m, "mtime")
+	btime, btimeOK := o.parseMetadataTime(m, "btime")
+	if atimeOK || mtimeOK {
+		if atimeOK && !mtimeOK {
+			mtime = atime
+		}
+		if !atimeOK && mtimeOK {
+			atime = mtime
+		}
+		err = o.setTimes(atime, mtime)
+		if err != nil {
+			outErr = fmt.Errorf("failed to set times: %w", err)
+		}
+	}
+	if haveSetBTime {
+		if btimeOK {
+			err = setBTime(o.path, btime)
+			if err != nil {
+				outErr = fmt.Errorf("failed to set birth (creation) time: %w", err)
+			}
+		}
+	}
+	uid, hasUID := o.parseMetadataInt(m, "uid", 10)
+	gid, hasGID := o.parseMetadataInt(m, "gid", 10)
+	if hasUID {
+		// FIXME should read UID and GID of current user and only attempt to set it if different
+		if !hasGID {
+			gid = uid
+		}
+		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
+			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
+		} else {
+			err = os.Chown(o.path, uid, gid)
+			if err != nil {
+				outErr = fmt.Errorf("failed to change ownership: %w", err)
+			}
+		}
+	}
+	mode, hasMode := o.parseMetadataInt(m, "mode", 8)
+	if hasMode {
+		err = os.Chmod(o.path, os.FileMode(mode))
+		if err != nil {
+			outErr = fmt.Errorf("failed to change permissions: %w", err)
+		}
+	}
+	// FIXME not parsing rdev yet
+	return outErr
+}

backend/local/metadata_bsd.go

@@ -0,0 +1,38 @@
+//go:build darwin || freebsd || netbsd
+// +build darwin freebsd netbsd
+
+package local
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	info, err := o.fs.lstat(o.path)
+	if err != nil {
+		return err
+	}
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		fs.Debugf(o, "didn't return Stat_t as expected")
+		return nil
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
+	}
+	setTime := func(key string, t syscall.Timespec) {
+		m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atimespec)
+	setTime("mtime", stat.Mtimespec)
+	setTime("btime", stat.Birthtimespec)
+	return nil
+}