vfs: disable --buffer-size read ahead

See: https://forum.rclone.org/t/for-mount-sftp-why-right-click-on-exe-file-is-so-slow-until-it-freezes/33830/52

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+github: [ncw]
+patreon: njcw
+liberapay: ncw
+custom: ["https://rclone.org/donate/"]

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"

.github/workflows/build.yml

@@ -8,31 +8,29 @@ name: build
 on:
   push:
     branches:
-      - '**'
+      - '*'
     tags:
-      - '**'
+      - '*'
   pull_request:
   workflow_dispatch:
     inputs:
       manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
         required: true
         default: true
 
 jobs:
   build:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.21'
+            go: '1.19'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +41,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.21'
+            go: '1.19'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.21'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -59,14 +57,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.21'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.21'
+            go: '1.19'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,20 +74,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.21'
+            go: '1.19'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.19
+          - job_name: go1.17
             os: ubuntu-latest
-            go: '1.19'
+            go: '1.17'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.20
+          - job_name: go1.18
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.18'
             quicktest: true
             racequicktest: true
 
@@ -99,12 +97,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
           check-latest: true
@@ -124,17 +122,12 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
+          sudo apt-get install fuse libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
         shell: bash
         run: |
-          # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
-          # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
-          unset HOMEBREW_NO_INSTALL_FROM_API
-          brew untap --force homebrew/core
-          brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
         if: matrix.os == 'macos-11'
@@ -168,7 +161,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -216,22 +209,23 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
+          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
           make ci_beta
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
         # Deploy binaries if enabled in config && not a PR && not a fork
-        if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
     name: "lint"
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Code quality test
         uses: golangci/golangci-lint-action@v3
@@ -241,9 +235,9 @@ jobs:
 
       # Run govulncheck on the latest go version, the one we build binaries with
       - name: Install Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: 1.19
           check-latest: true
 
       - name: Install govulncheck
@@ -253,25 +247,25 @@ jobs:
         run: govulncheck ./...
 
   android:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
     name: "android-all"
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: 1.19
 
       - name: Go module cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -356,4 +350,4 @@ jobs:
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork
-        if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_beta_docker_image.yml

@@ -1,77 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKERHUB_USERNAME }}
-                password: ${{ secrets.DOCKERHUB_TOKEN }}
-            - name: Extract metadata (tags, labels) for Docker
-              id: meta
-              uses: docker/metadata-action@v5
-              with:
-                images: ghcr.io/${{ github.repository }}
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to GitHub Container Registry
-              uses: docker/login-action@v3
-              with:
-                registry: ghcr.io
-                # This is the user that triggered the Workflow. In this case, it will
-                # either be the user whom created the Release or manually triggered
-                # the workflow_dispatch.
-                username: ${{ github.actor }}
-                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
-                # GitHub Actions at the start of a workflow run to identify the job.
-                # This is used to authenticate against GitHub Container Registry.
-                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
-                # for more detailed information.
-                password: ${{ secrets.GITHUB_TOKEN }}
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .
-            - name: Build and publish image
-              uses: docker/build-push-action@v5
-              with:
-                file: Dockerfile
-                context: .
-                push: true # push the image to ghcr
-                tags: |
-                  ghcr.io/rclone/rclone:beta
-                  rclone/rclone:beta
-                labels: ${{ steps.meta.outputs.labels }}
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                cache-from: type=gha, scope=${{ github.workflow }}
-                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
-                provenance: false
-              # Eventually cache will need to be cleared if builds more frequent than once a week
-              # https://github.com/docker/build-push-action/issues/252
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,26 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@1.1.0
+              with:
+                tag: beta
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -10,17 +10,8 @@ jobs:
         runs-on: ubuntu-latest
         name: Build image job
         steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
             - name: Checkout master
-              uses: actions/checkout@v4
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -48,17 +39,8 @@ jobs:
         runs-on: ubuntu-latest
         name: Build docker plugin job
         steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
             - name: Checkout master
-              uses: actions/checkout@v4
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

.github/workflows/winget.yml

@@ -1,14 +0,0 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}

.gitignore

@@ -8,13 +8,10 @@ rclone.iml
 .idea
 .history
 *.test
+*.log
 *.iml
 fuzz-build.zip
 *.orig
 *.rej
 Thumbs.db
 __pycache__
-.DS_Store
-/docs/static/img/logos/
-resource_windows_*.syso
-.devcontainer

.golangci.yml

@@ -2,17 +2,15 @@
 
 linters:
   enable:
+    - deadcode
     - errcheck
     - goimports
     - revive
     - ineffassign
+    - structcheck
+    - varcheck
     - govet
     - unconvert
-    - staticcheck
-    - gosimple
-    - stylecheck
-    - unused
-    - misspell
     #- prealloc
     #- maligned
   disable-all: true
@@ -27,74 +25,6 @@ issues:
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
-  exclude-rules:
-
-    - linters:
-      - staticcheck
-      text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
-
-  # don't disable the revive messages about comments on exported functions
-  include:
-    - EXC0012
-    - EXC0013
-    - EXC0014
-    - EXC0015
-
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 10m
-
-linters-settings:
-  revive:
-    # setting rules seems to disable all the rules, so re-enable them here
-    rules:
-      - name: blank-imports
-        disabled: false
-      - name: context-as-argument
-        disabled: false
-      - name: context-keys-type
-        disabled: false
-      - name: dot-imports
-        disabled: false
-      - name: empty-block
-        disabled: true
-      - name: error-naming
-        disabled: false
-      - name: error-return
-        disabled: false
-      - name: error-strings
-        disabled: false
-      - name: errorf
-        disabled: false
-      - name: exported
-        disabled: false
-      - name: increment-decrement
-        disabled: true
-      - name: indent-error-flow
-        disabled: false
-      - name: package-comments
-        disabled: false
-      - name: range
-        disabled: false
-      - name: receiver-naming
-        disabled: false
-      - name: redefines-builtin-id
-        disabled: true
-      - name: superfluous-else
-        disabled: true
-      - name: time-naming
-        disabled: false
-      - name: unexported-return
-        disabled: false
-      - name: unreachable-code
-        disabled: true
-      - name: unused-parameter
-        disabled: true
-      - name: var-declaration
-        disabled: false
-      - name: var-naming
-        disabled: false
-  stylecheck:
-    # Only enable the checks performed by the staticcheck stand-alone tool,
-    # as documented here: https://staticcheck.io/docs/configuration/options/#checks
-    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]

CONTRIBUTING.md

@@ -1,8 +1,8 @@
-# Contributing to rclone
+# Contributing to rclone #
 
 This is a short guide on how to contribute things to rclone.
 
-## Reporting a bug
+## Reporting a bug ##
 
 If you've just got a question or aren't sure if you've found a bug
 then please use the [rclone forum](https://forum.rclone.org/) instead
@@ -12,13 +12,13 @@ When filing an issue, please include the following information if
 possible as well as a description of the problem.  Make sure you test
 with the [latest beta of rclone](https://beta.rclone.org/):
 
-- Rclone version (e.g. output from `rclone version`)
-- Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
-- The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
-- A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
-    - if the log contains secrets then edit the file with a text editor first to obscure them
+  * Rclone version (e.g. output from `rclone version`)
+  * Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
+  * The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
+  * A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
+    * if the log contains secrets then edit the file with a text editor first to obscure them
 
-## Submitting a new feature or bug fix
+## Submitting a new feature or bug fix ##
 
 If you find a bug that you'd like to fix, or a new feature that you'd
 like to implement then please submit a pull request via GitHub.
@@ -73,9 +73,9 @@ This is typically enough if you made a simple bug fix, otherwise please read the
 
 Make sure you
 
-- Add [unit tests](#testing) for a new feature.
-- Add [documentation](#writing-documentation) for a new feature.
-- [Commit your changes](#committing-your-changes) using the [commit message guidelines](#commit-messages).
+  * Add [unit tests](#testing) for a new feature.
+  * Add [documentation](#writing-documentation) for a new feature.
+  * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
 
 When you are done with that push your changes to GitHub:
 
@@ -88,9 +88,9 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I
 
 You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
 
-## Using Git and GitHub
+## Using Git and GitHub ##
 
-### Committing your changes
+### Committing your changes ###
 
 Follow the guideline for [commit messages](#commit-messages) and then:
 
@@ -107,7 +107,7 @@ You can modify the message or changes in the latest commit using:
 
 If you amend to commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
 
-### Replacing your previously pushed commits
+### Replacing your previously pushed commits ###
 
 Note that you are about to rewrite the GitHub history of your branch. It is good practice to involve your collaborators before modifying commits that have been pushed to GitHub.
 
@@ -115,7 +115,7 @@ Your previously pushed commits are replaced by:
 
     git push --force origin my-new-feature 
 
-### Basing your changes on the latest master
+### Basing your changes on the latest master ###
 
 To base your changes on the latest version of the [rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
 
@@ -149,21 +149,13 @@ If you squash commits that have been pushed to GitHub, then you will have to [re
 
 Tip: You may like to use `git rebase -i master` if you are experienced or have a more complex situation.
 
-### GitHub Continuous Integration
+### GitHub Continuous Integration ###
 
 rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
-## Testing
+## Testing ##
 
-### Code quality tests
-
-If you install [golangci-lint](https://github.com/golangci/golangci-lint) then you can run the same tests as get run in the CI which can be very helpful.
-
-You can run them with `make check` or with `golangci-lint run ./...`.
-
-Using these tests ensures that the rclone codebase all uses the same coding standards. These tests also check for easy mistakes to make (like forgetting to check an error return).
-
-### Quick testing
+### Quick testing ###
 
 rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
@@ -176,7 +168,7 @@ You can also use `make`, if supported by your platform
 
 The quicktest is [automatically run by GitHub](#github-continuous-integration) when you push your branch to GitHub.
 
-### Backend testing
+### Backend testing ###
 
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
@@ -211,7 +203,7 @@ project root:
     go install github.com/rclone/rclone/fstest/test_all
     test_all -backend drive
 
-### Full integration testing
+### Full integration testing ###
 
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
@@ -226,56 +218,55 @@ The commands may require some extra go packages which you can install with
 The full integration tests are run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
-## Code Organisation
+## Code Organisation ##
 
 Rclone code is organised into a small number of top level directories
 with modules beneath.
 
-- backend - the rclone backends for interfacing to cloud providers -
-    - all - import this to load all the cloud providers
-    - ...providers
-- bin - scripts for use while building or maintaining rclone
-- cmd - the rclone commands
-    - all - import this to load all the commands
-    - ...commands
-- cmdtest - end-to-end tests of commands, flags, environment variables,...
-- docs - the documentation and website
-    - content - adjust these docs only - everything else is autogenerated
-    - command - these are auto-generated - edit the corresponding .go file
-- fs - main rclone definitions - minimal amount of code
-    - accounting - bandwidth limiting and statistics
-    - asyncreader - an io.Reader which reads ahead
-    - config - manage the config file and flags
-    - driveletter - detect if a name is a drive letter
-    - filter - implements include/exclude filtering
-    - fserrors - rclone specific error handling
-    - fshttp - http handling for rclone
-    - fspath - path handling for rclone
-    - hash - defines rclone's hash types and functions
-    - list - list a remote
-    - log - logging facilities
-    - march - iterates directories in lock step
-    - object - in memory Fs objects
-    - operations - primitives for sync, e.g. Copy, Move
-    - sync - sync directories
-    - walk - walk a directory
-- fstest - provides integration test framework
-    - fstests - integration tests for the backends
-    - mockdir - mocks an fs.Directory
-    - mockobject - mocks an fs.Object
-    - test_all - Runs integration tests for everything
-- graphics - the images used in the website, etc.
-- lib - libraries used by the backend
-    - atexit - register functions to run when rclone exits
-    - dircache - directory ID to name caching
-    - oauthutil - helpers for using oauth
-    - pacer - retries with backoff and paces operations
-    - readers - a selection of useful io.Readers
-    - rest - a thin abstraction over net/http for REST
-- librclone - in memory interface to rclone's API for embedding rclone
-- vfs - Virtual FileSystem layer for implementing rclone mount and similar
+  * backend - the rclone backends for interfacing to cloud providers -
+    * all - import this to load all the cloud providers
+    * ...providers
+  * bin - scripts for use while building or maintaining rclone
+  * cmd - the rclone commands
+    * all - import this to load all the commands
+    * ...commands
+  * cmdtest - end-to-end tests of commands, flags, environment variables,...
+  * docs - the documentation and website
+    * content - adjust these docs only - everything else is autogenerated
+      * command - these are auto-generated - edit the corresponding .go file
+  * fs - main rclone definitions - minimal amount of code
+    * accounting - bandwidth limiting and statistics
+    * asyncreader - an io.Reader which reads ahead
+    * config - manage the config file and flags
+    * driveletter - detect if a name is a drive letter
+    * filter - implements include/exclude filtering
+    * fserrors - rclone specific error handling
+    * fshttp - http handling for rclone
+    * fspath - path handling for rclone
+    * hash - defines rclone's hash types and functions
+    * list - list a remote
+    * log - logging facilities
+    * march - iterates directories in lock step
+    * object - in memory Fs objects
+    * operations - primitives for sync, e.g. Copy, Move
+    * sync - sync directories
+    * walk - walk a directory
+  * fstest - provides integration test framework
+    * fstests - integration tests for the backends
+    * mockdir - mocks an fs.Directory
+    * mockobject - mocks an fs.Object
+    * test_all - Runs integration tests for everything
+  * graphics - the images used in the website, etc.
+  * lib - libraries used by the backend
+    * atexit - register functions to run when rclone exits
+    * dircache - directory ID to name caching
+    * oauthutil - helpers for using oauth
+    * pacer - retries with backoff and paces operations
+    * readers - a selection of useful io.Readers
+    * rest - a thin abstraction over net/http for REST
+  * vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
-## Writing Documentation
+## Writing Documentation ##
 
 If you are adding a new feature then please update the documentation.
 
@@ -286,22 +277,22 @@ alphabetical order.
 If you add a new backend option/flag, then it should be documented in
 the source file in the `Help:` field.
 
-- Start with the most important information about the option,
+  * Start with the most important information about the option,
     as a single sentence on a single line.
-    - This text will be used for the command-line flag help.
-    - It will be combined with other information, such as any default value,
+    * This text will be used for the command-line flag help.
+    * It will be combined with other information, such as any default value,
       and the result will look odd if not written as a single sentence.
-    - It should end with a period/full stop character, which will be shown
+    * It should end with a period/full stop character, which will be shown
       in docs but automatically removed when producing the flag help.
-    - Try to keep it below 80 characters, to reduce text wrapping in the terminal.
-- More details can be added in a new paragraph, after an empty line (`"\n\n"`).
-    - Like with docs generated from Markdown, a single line break is ignored
+    * Try to keep it below 80 characters, to reduce text wrapping in the terminal.
+  * More details can be added in a new paragraph, after an empty line (`"\n\n"`).
+    * Like with docs generated from Markdown, a single line break is ignored
       and two line breaks creates a new paragraph.
-    - This text will be shown to the user in `rclone config`
+    * This text will be shown to the user in `rclone config`
       and in the docs (where it will be added by `make backenddocs`,
       normally run some time before next release).
-- To create options of enumeration type use the `Examples:` field.
-    - Each example value have their own `Help:` field, but they are treated
+  * To create options of enumeration type use the `Examples:` field.
+    * Each example value have their own `Help:` field, but they are treated
       a bit different than the main option help text. They will be shown
       as an unordered list, therefore a single line break is enough to
       create a new list item. Also, for enumeration texts like name of
@@ -321,12 +312,12 @@ combined unmodified with other information (such as any default value).
 Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
 for small changes in the docs which makes it very easy.
 
-## Making a release
+## Making a release ##
 
 There are separate instructions for making a release in the RELEASE.md
 file.
 
-## Commit messages
+## Commit messages ##
 
 Please make the first line of your commit message a summary of the
 change that a user (not a developer) of rclone would like to read, and
@@ -367,7 +358,7 @@ error fixing the hang.
 Fixes #1498
 ```
 
-## Adding a dependency
+## Adding a dependency ##
 
 rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
@@ -379,7 +370,7 @@ To add a dependency `github.com/ncw/new_dependency` see the
 instructions below.  These will fetch the dependency and add it to
 `go.mod` and `go.sum`.
 
-    go get github.com/ncw/new_dependency
+    GO111MODULE=on go get github.com/ncw/new_dependency
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
@@ -387,15 +378,15 @@ go docs linked above), but don't unless you really need to.
 Please check in the changes generated by `go mod` including `go.mod`
 and `go.sum` in the same commit as your other changes.
 
-## Updating a dependency
+## Updating a dependency ##
 
 If you need to update a dependency then run
 
-    go get golang.org/x/crypto
+    GO111MODULE=on go get -u golang.org/x/crypto
 
 Check in a single commit as above.
 
-## Updating all the dependencies
+## Updating all the dependencies ##
 
 In order to update all the dependencies then run `make update`.  This
 just uses the go modules to update all the modules to their latest
@@ -404,7 +395,7 @@ stable release. Check in the changes in a single commit as above.
 This should be done early in the release cycle to pick up new versions
 of packages in time for them to get some testing.
 
-## Updating a backend
+## Updating a backend ##
 
 If you update a backend then please run the unit tests and the
 integration tests for that backend.
@@ -419,133 +410,82 @@ integration tests.
 
 The next section goes into more detail about the tests.
 
-## Writing a new backend
+## Writing a new backend ##
 
 Choose a name.  The docs here will use `remote` as an example.
 
 Note that in rclone terminology a file system backend is called a
 remote or an fs.
 
-### Research
+Research
 
-- Look at the interfaces defined in `fs/types.go`
-- Study one or more of the existing remotes
+  * Look at the interfaces defined in `fs/fs.go`
+  * Study one or more of the existing remotes
 
-### Getting going
+Getting going
 
-- Create `backend/remote/remote.go` (copy this from a similar remote)
-    - box is a good one to start from if you have a directory-based remote (and shows how to use the directory cache)
-    - b2 is a good one to start from if you have a bucket-based remote
-- Add your remote to the imports in `backend/all/all.go`
-- HTTP based remotes are easiest to maintain if they use rclone's [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) module, but if there is a really good Go SDK from the provider then use that instead.
-- Try to implement as many optional methods as possible as it makes the remote more usable.
-- Use [lib/encoder](https://pkg.go.dev/github.com/rclone/rclone/lib/encoder) to make sure we can encode any path name and `rclone info` to help determine the encodings needed
-    - `rclone purge -v TestRemote:rclone-info`
-    - `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    - `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
-    - open `remote.csv` in a spreadsheet and examine
+  * Create `backend/remote/remote.go` (copy this from a similar remote)
+    * box is a good one to start from if you have a directory-based remote
+    * b2 is a good one to start from if you have a bucket-based remote
+  * Add your remote to the imports in `backend/all/all.go`
+  * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
+  * Try to implement as many optional methods as possible as it makes the remote more usable.
+  * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
+    * `rclone purge -v TestRemote:rclone-info`
+    * `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    * `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
+    * open `remote.csv` in a spreadsheet and examine
 
-### Guidelines for a speedy merge
+Unit tests
 
-- **Do** use [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) if you are implementing a REST like backend and parsing XML/JSON in the backend.
-- **Do** use rclone's Client or Transport from [fs/fshttp](https://pkg.go.dev/github.com/rclone/rclone/fs/fshttp) if your backend is HTTP based - this adds features like `--dump bodies`, `--tpslimit`, `--user-agent` without you having to code anything!
-- **Do** follow your example backend exactly - use the same code order, function names, layout, structure. **Don't** move stuff around and **Don't** delete the comments.
-- **Do not** split your backend up into `fs.go` and `object.go` (there are a few backends like that - don't follow them!)
-- **Do** put your API type definitions in a separate file - by preference `api/types.go`
-- **Remember** we have >50 backends to maintain so keeping them as similar as possible to each other is a high priority!
+  * Create a config entry called `TestRemote` for the unit tests to use
+  * Create a `backend/remote/remote_test.go` - copy and adjust your example remote
+  * Make sure all tests pass with `go test -v`
 
-### Unit tests
+Integration tests
 
-- Create a config entry called `TestRemote` for the unit tests to use
-- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
-- Make sure all tests pass with `go test -v`
-
-### Integration tests
-
-- Add your backend to `fstest/test_all/config.yaml`
-    - Once you've done that then you can use the integration test framework from the project root:
-    - go install ./...
-    - test_all -backends remote
+  * Add your backend to `fstest/test_all/config.yaml`
+      * Once you've done that then you can use the integration test framework from the project root:
+      * go install ./...
+      * test_all -backends remote
 
 Or if you want to run the integration tests manually:
 
-- Make sure integration tests pass with
-    - `cd fs/operations`
-    - `go test -v -remote TestRemote:`
-    - `cd fs/sync`
-    - `go test -v -remote TestRemote:`
-- If your remote defines `ListR` check with this also
-    - `go test -v -remote TestRemote: -fast-list`
+  * Make sure integration tests pass with
+      * `cd fs/operations`
+      * `go test -v -remote TestRemote:`
+      * `cd fs/sync`
+      * `go test -v -remote TestRemote:`
+  * If your remote defines `ListR` check with this also
+      * `go test -v -remote TestRemote: -fast-list`
 
 See the [testing](#testing) section for more information on integration tests.
 
-### Backend documentation
-
-Add your backend to the docs - you'll need to pick an icon for it from
+Add your fs to the docs - you'll need to pick an icon for it from
 [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
 alphabetical order of full name of remote (e.g. `drive` is ordered as
 `Google Drive`) but with the local file system last.
 
-- `README.md` - main GitHub page
-- `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
-    - make sure this has the `autogenerated options` comments in (see your reference backend docs)
-    - update them in your backend with `bin/make_backend_docs.py remote`
-- `docs/content/overview.md` - overview docs
-- `docs/content/docs.md` - list of remotes in config section
-- `docs/content/_index.md` - front page of rclone.org
-- `docs/layouts/chrome/navbar.html` - add it to the website navigation
-- `bin/make_manual.py` - add the page to the `docs` constant
+  * `README.md` - main GitHub page
+  * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+    * make sure this has the `autogenerated options` comments in (see your reference backend docs)
+    * update them with `make backenddocs` - revert any changes in other backends
+  * `docs/content/overview.md` - overview docs
+  * `docs/content/docs.md` - list of remotes in config section
+  * `docs/content/_index.md` - front page of rclone.org
+  * `docs/layouts/chrome/navbar.html` - add it to the website navigation
+  * `bin/make_manual.py` - add the page to the `docs` constant
 
 Once you've written the docs, run `make serve` and check they look OK
 in the web browser and the links (internal and external) all work.
 
-## Adding a new s3 provider
-
-It is quite easy to add a new S3 provider to rclone.
-
-You'll need to modify the following files
-
-- `backend/s3/s3.go`
-    - Add the provider to `providerOption` at the top of the file
-    - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
-    - Exclude your provider from genric config questions (eg `region` and `endpoint).
-    - Add the provider to the `setQuirks` function - see the documentation there.
-- `docs/content/s3.md`
-    - Add the provider at the top of the page.
-    - Add a section about the provider linked from there.
-    - Add a transcript of a trial `rclone config` session
-        - Edit the transcript to remove things which might change in subsequent versions
-    - **Do not** alter or add to the autogenerated parts of `s3.md`
-    - **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
-- `README.md` - this is the home page in github
-    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
-- `docs/content/_index.md` - this is the home page of rclone.org
-    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
-
-When adding the provider, endpoints, quirks, docs etc keep them in
-alphabetical order by `Provider` name, but with `AWS` first and
-`Other` last.
-
-Once you've written the docs, run `make serve` and check they look OK
-in the web browser and the links (internal and external) all work.
-
-Once you've written the code, test `rclone config` works to your
-satisfaction, and check the integration tests work `go test -v -remote
-NewS3Provider:`. You may need to adjust the quirks to get them to
-pass. Some providers just can't pass the tests with control characters
-in the names so if these fail and the provider doesn't support
-`urlEncodeListings` in the quirks then ignore them. Note that the
-`SetTier` test may also fail on non AWS providers.
-
-For an example of adding an s3 provider see [eb3082a1](https://github.com/rclone/rclone/commit/eb3082a1ebdb76d5625f14cedec3f5154a5e7b10).
-
-## Writing a plugin
+## Writing a plugin ##
 
 New features (backends, commands) can also be added "out-of-tree", through Go plugins.
 Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
 This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
 
-### Usage
+Usage
 
  - Naming
    - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
@@ -560,7 +500,7 @@ This is useful if you can't merge your changes upstream or don't want to maintai
    - Plugins must be compiled against the exact version of rclone to work.
      (The rclone used during building the plugin must be the same as the source of rclone)
 
-### Building
+Building
 
 To turn your existing additions into a Go plugin, move them to an external repository
 and change the top-level package name to `main`.

Dockerfile

@@ -1,9 +1,8 @@
-FROM golang:alpine AS builder
+FROM golang AS builder
 
 COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN apk add --no-cache make bash gawk git
 RUN \
   CGO_ENABLED=0 \
   make
@@ -12,7 +11,7 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse3 tzdata && \
+RUN apk --no-cache add ca-certificates fuse tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

MAINTAINERS.md

@@ -16,11 +16,6 @@ Current active maintainers of rclone are:
 | Max Sum          | @Max-Sum          | union backend                |
 | Fred             | @creativeprojects | seafile backend              |
 | Caleb Case       | @calebcase        | storj backend                |
-| wiserain         | @wiserain         | pikpak backend               |
-| albertony        | @albertony        |                              |
-| Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
-| Hideo Aoyama     | @boukendesho      | snap packaging               |
-| nielash          | @nielash          | bisync                       |
 
 **This is a work in progress Draft**
 

Makefile

@@ -30,7 +30,6 @@ ifdef RELEASE_TAG
 	TAG := $(RELEASE_TAG)
 endif
 GO_VERSION := $(shell go version)
-GO_OS := $(shell go env GOOS)
 ifdef BETA_SUBDIR
 	BETA_SUBDIR := /$(BETA_SUBDIR)
 endif
@@ -47,13 +46,7 @@ endif
 .PHONY: rclone test_all vars version
 
 rclone:
-ifeq ($(GO_OS),windows)
-	go run bin/resource_windows.go -version $(TAG) -syso resource_windows_`go env GOARCH`.syso
-endif
 	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
-ifeq ($(GO_OS),windows)
-	rm resource_windows_`go env GOARCH`.syso
-endif
 	mkdir -p `go env GOPATH`/bin/
 	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
 	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
@@ -73,10 +66,6 @@ btest:
 	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
 	@echo "Copied markdown of beta release to clip board"
 
-btesth:
-	@echo "<a href="$(BETA_URL)">$(TAG)</a> on branch <a href="https://github.com/rclone/rclone/tree/$(BRANCH)">$(BRANCH)</a> (uploaded in 15-30 mins)" | xclip -r -sel clip -t text/html
-	@echo "Copied beta release in HTML to clip board"
-
 version:
 	@echo '$(TAG)'
 
@@ -92,9 +81,6 @@ quicktest:
 racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
-compiletest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
-
 # Do source code quality checks
 check:	rclone
 	@echo "-- START CODE QUALITY REPORT -------------------------------"
@@ -107,7 +93,11 @@ build_dep:
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
-	go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64\.tar\.gz'
+
+# Get the release dependencies we only install on Windows
+release_dep_windows:
+	GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
 
 # Update dependencies
 showupdates:
@@ -153,7 +143,7 @@ rcdocs: rclone
 
 install: rclone
 	install -d ${DESTDIR}/usr/bin
-	install ${GOPATH}/bin/rclone ${DESTDIR}/usr/bin
+	install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone
 
 clean:
 	go clean ./...

README.md

@@ -1,7 +1,5 @@
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
-[<img src="https://rclone.org/img/logos/warp-github-light.svg" title="Visit warp.dev to learn more." align="right">](https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103#gh-light-mode-only)
-[<img src="https://rclone.org/img/logos/warp-github-dark.svg" title="Visit warp.dev to learn more." align="right">](https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103#gh-dark-mode-only)
 
 [Website](https://rclone.org) |
 [Documentation](https://rclone.org/docs/) |
@@ -27,19 +25,18 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
-  * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
   * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
+  * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
   * Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
-  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
@@ -48,41 +45,30 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
-  * ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
-  * Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
-  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
-  * Linkbox [:page_facing_up:](https://rclone.org/linkbox)
-  * Linode Object Storage [:page_facing_up:](https://rclone.org/s3/#linode)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
-  * Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
   * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
-  * Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
-  * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
-  * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
-  * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
-  * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
@@ -93,7 +79,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * Storj [:page_facing_up:](https://rclone.org/storj/)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
-  * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -123,7 +108,6 @@ These backends adapt or modify other storage providers
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
-  * [Bisync](https://rclone.org/bisync/) (two way) to keep two directories in sync bidirectionally
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
   * Optional large file chunking ([Chunker](https://rclone.org/chunker/))

RELEASE.md

@@ -10,7 +10,7 @@ This file describes how to make the various kinds of releases
 ## Making a release
 
   * git checkout master # see below for stable branch
-  * git pull # IMPORTANT
+  * git pull
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
@@ -21,7 +21,6 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
-  * git push origin # without --follow-tags so it doesn't push the tag if it fails
   * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
@@ -41,15 +40,12 @@ Early in the next release cycle update the dependencies
 
   * Review any pinned packages in go.mod and remove if possible
   * make updatedirect
-  * make GOTAGS=cmount
-  * make compiletest
+  * make
   * git commit -a -v
   * make update
-  * make GOTAGS=cmount
-  * make compiletest
+  * make
   * roll back any updates which didn't compile
   * git commit -a -v --amend
-  * **NB** watch out for this changing the default go version in `go.mod`
 
 Note that `make update` updates all direct and indirect dependencies
 and there can occasionally be forwards compatibility problems with
@@ -78,7 +74,8 @@ Set vars
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
+  * git branch ${BASE_TAG} ${BASE_TAG}-stable
+  * git co ${BASE_TAG}-stable
   * make startstable
 
 Now
@@ -93,52 +90,34 @@ Now
   * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
   * git push
 
-## Sponsor logos
-
-If updating the website note that the sponsor logos have been moved out of the main repository.
-
-You will need to checkout `/docs/static/img/logos` from https://github.com/rclone/third-party-logos
-which is a private repo containing artwork from sponsors.
-
-## Update the website between releases
-
-Create an update website branch based off the last release
-
-    git co -b update-website
-
-If the branch already exists, double check there are no commits that need saving.
-
-Now reset the branch to the last release
-
-    git reset --hard v1.64.0
-
-Create the changes, check them in, test with `make serve` then
-
-    make upload_test_website
-
-Check out https://test.rclone.org and when happy
-
-    make upload_website
-
-Cherry pick any changes back to master and the stable branch if it is active.
-
 ## Making a manual build of docker
 
-To do a basic build of rclone's docker image to debug builds locally:
-
-```
-docker buildx build --load -t rclone/rclone:testing --progress=plain .
-docker run --rm rclone/rclone:testing version
-```
-
-To test the multipatform build
-
-```
-docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 .
-```
-
-To make a full build then set the tags correctly and add `--push`
+The rclone docker image should autobuild on via GitHub actions.  If it doesn't
+or needs to be updated then rebuild like this.
+
+See: https://github.com/ilteoood/docker_buildx/issues/19
+See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
 
 ```
+git co v1.54.1
+docker pull golang
+export DOCKER_CLI_EXPERIMENTAL=enabled
+docker buildx create --name actions_builder --use
+docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
+echo "Supported platforms: $SUPPORTED_PLATFORMS"
 docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
+docker buildx stop actions_builder
+```
+
+### Old build for linux/amd64 only
+
+```
+docker pull golang
+docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
+docker push rclone/rclone:1.52.0
+docker push rclone/rclone:1.52
+docker push rclone/rclone:1
+docker push rclone/rclone:latest
 ```

VERSION

@@ -1 +1 @@
-v1.65.2
+v1.61.0

backend/all/all.go

@@ -6,7 +6,6 @@ import (
 	_ "github.com/rclone/rclone/backend/alias"
 	_ "github.com/rclone/rclone/backend/amazonclouddrive"
 	_ "github.com/rclone/rclone/backend/azureblob"
-	_ "github.com/rclone/rclone/backend/azurefiles"
 	_ "github.com/rclone/rclone/backend/b2"
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
@@ -25,11 +24,9 @@ import (
 	_ "github.com/rclone/rclone/backend/hdfs"
 	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/imagekit"
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
-	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/mailru"
 	_ "github.com/rclone/rclone/backend/mega"
@@ -39,12 +36,9 @@ import (
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
-	_ "github.com/rclone/rclone/backend/pikpak"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
-	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
-	_ "github.com/rclone/rclone/backend/quatrix"
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/seafile"
 	_ "github.com/rclone/rclone/backend/sftp"

backend/azureblob/azureblob_internal_test.go

@@ -17,3 +17,20 @@ func (f *Fs) InternalTest(t *testing.T) {
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+func TestIncrement(t *testing.T) {
+	for _, test := range []struct {
+		in   []byte
+		want []byte
+	}{
+		{[]byte{0, 0, 0, 0}, []byte{1, 0, 0, 0}},
+		{[]byte{0xFE, 0, 0, 0}, []byte{0xFF, 0, 0, 0}},
+		{[]byte{0xFF, 0, 0, 0}, []byte{0, 1, 0, 0}},
+		{[]byte{0, 1, 0, 0}, []byte{1, 1, 0, 0}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFE}, []byte{0, 0, 0, 0xFF}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFF}, []byte{0, 0, 0, 0}},
+	} {
+		increment(test.in)
+		assert.Equal(t, test.want, test.in)
+	}
+}

backend/azureblob/azureblob_test.go

@@ -6,10 +6,10 @@
 package azureblob
 
 import (
+	"context"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
 )
@@ -19,29 +19,8 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:    "TestAzureBlob:",
 		NilObject:     (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool", "Cold"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: defaultChunkSize,
-		},
-	})
-}
-
-// TestIntegration2 runs integration tests against the remote
-func TestIntegration2(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	name := "TestAzureBlob"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:  name + ":",
-		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool", "Cold"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: defaultChunkSize,
-		},
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "directory_markers", Value: "true"},
-		},
+		TiersToTest:   []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{},
 	})
 }
 
@@ -53,6 +32,36 @@ var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 )
 
+// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
+func TestServicePrincipalFileSuccess(t *testing.T) {
+	ctx := context.TODO()
+	credentials := `
+{
+    "appId": "my application (client) ID",
+    "password": "my secret",
+    "tenant": "my active directory tenant ID"
+}
+`
+	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	if assert.NoError(t, err) {
+		assert.NotNil(t, tokenRefresher)
+	}
+}
+
+// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
+func TestServicePrincipalFileFailure(t *testing.T) {
+	ctx := context.TODO()
+	credentials := `
+{
+    "appId": "my application (client) ID",
+    "tenant": "my active directory tenant ID"
+}
+`
+	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	assert.Error(t, err)
+	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
+}
+
 func TestValidateAccessTier(t *testing.T) {
 	tests := map[string]struct {
 		accessTier string
@@ -62,7 +71,6 @@ func TestValidateAccessTier(t *testing.T) {
 		"HOT":     {"HOT", true},
 		"Hot":     {"Hot", true},
 		"cool":    {"cool", true},
-		"cold":    {"cold", true},
 		"archive": {"archive", true},
 		"empty":   {"", false},
 		"unknown": {"unknown", false},

backend/azureblob/imds.go

@@ -0,0 +1,136 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package azureblob
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
+)
+
+const (
+	azureResource      = "https://storage.azure.com"
+	imdsAPIVersion     = "2018-02-01"
+	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
+)
+
+// This custom type is used to add the port the test server has bound to
+// to the request context.
+type testPortKey string
+
+type msiIdentifierType int
+
+const (
+	msiClientID msiIdentifierType = iota
+	msiObjectID
+	msiResourceID
+)
+
+type userMSI struct {
+	Type  msiIdentifierType
+	Value string
+}
+
+type httpError struct {
+	Response *http.Response
+}
+
+func (e httpError) Error() string {
+	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
+}
+
+// GetMSIToken attempts to obtain an MSI token from the Azure Instance
+// Metadata Service.
+func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
+	// Attempt to get an MSI token; silently continue if unsuccessful.
+	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
+	result := adal.Token{}
+	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
+	if err != nil {
+		fs.Debugf(nil, "Failed to create request: %v", err)
+		return result, err
+	}
+	params := req.URL.Query()
+	params.Set("resource", azureResource)
+	params.Set("api-version", imdsAPIVersion)
+
+	// Specify user-assigned identity if requested.
+	if identity != nil {
+		switch identity.Type {
+		case msiClientID:
+			params.Set("client_id", identity.Value)
+		case msiObjectID:
+			params.Set("object_id", identity.Value)
+		case msiResourceID:
+			params.Set("mi_res_id", identity.Value)
+		default:
+			// If this happens, the calling function and this one don't agree on
+			// what valid ID types exist.
+			return result, fmt.Errorf("unknown MSI identity type specified")
+		}
+	}
+	req.URL.RawQuery = params.Encode()
+
+	// The Metadata header is required by all calls to IMDS.
+	req.Header.Set("Metadata", "true")
+
+	// If this function is run in a test, query the test server instead of IMDS.
+	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
+	if isTest {
+		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
+		req.Host = req.URL.Host
+	}
+
+	// Send request
+	httpClient := fshttp.NewClient(ctx)
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
+	}
+	defer func() { // resp and Body should not be nil
+		_, err = io.Copy(io.Discard, resp.Body)
+		if err != nil {
+			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
+		}
+		err = resp.Body.Close()
+		if err != nil {
+			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
+		}
+	}()
+	// Check if the status code indicates success
+	// The request returns 200 currently, add 201 and 202 as well for possible extension.
+	switch resp.StatusCode {
+	case 200, 201, 202:
+		break
+	default:
+		body, _ := io.ReadAll(resp.Body)
+		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
+		return result, httpError{Response: resp}
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
+	}
+	// Remove BOM, if any. azcopy does this so I'm following along.
+	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
+
+	// This would be a good place to persist the token if a large number of rclone
+	// invocations are being made in a short amount of time. If the token is
+	// persisted, the azureblob code will need to check for expiry before every
+	// storage API call.
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err)
+	}
+
+	return result, nil
+}

backend/azureblob/imds_test.go

@@ -0,0 +1,118 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package azureblob
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		err := r.ParseForm()
+		require.NoError(t, err)
+		parameters := r.URL.Query()
+		(*actual)["path"] = r.URL.Path
+		(*actual)["Metadata"] = r.Header.Get("Metadata")
+		(*actual)["method"] = r.Method
+		for paramName := range parameters {
+			(*actual)[paramName] = parameters.Get(paramName)
+		}
+		// Make response.
+		response := adal.Token{}
+		responseBytes, err := json.Marshal(response)
+		require.NoError(t, err)
+		_, err = w.Write(responseBytes)
+		require.NoError(t, err)
+	}
+}
+
+func TestManagedIdentity(t *testing.T) {
+	// test user-assigned identity specifiers to use
+	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
+	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
+	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
+	tests := []struct {
+		identity              *userMSI
+		identityParameterName string
+		expectedAbsent        []string
+	}{
+		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
+		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
+		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
+		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
+	}
+	alwaysExpected := map[string]string{
+		"path":        "/metadata/identity/oauth2/token",
+		"resource":    "https://storage.azure.com",
+		"Metadata":    "true",
+		"api-version": "2018-02-01",
+		"method":      "GET",
+	}
+	for _, test := range tests {
+		actual := make(map[string]string, 10)
+		testServer := httptest.NewServer(handler(t, &actual))
+		defer testServer.Close()
+		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
+		require.NoError(t, err)
+		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
+		_, err = GetMSIToken(ctx, test.identity)
+		require.NoError(t, err)
+
+		// Validate expected query parameters present
+		expected := make(map[string]string)
+		for k, v := range alwaysExpected {
+			expected[k] = v
+		}
+		if test.identity != nil {
+			expected[test.identityParameterName] = test.identity.Value
+		}
+
+		for key := range expected {
+			value, exists := actual[key]
+			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
+				test.identityParameterName, key) {
+				assert.Equalf(t, expected[key], value,
+					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
+			}
+		}
+
+		// Validate unexpected query parameters absent
+		for _, key := range test.expectedAbsent {
+			_, exists := actual[key]
+			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
+		}
+	}
+}
+
+func errorHandler(resultCode int) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Test error generated", resultCode)
+	}
+}
+
+func TestIMDSErrors(t *testing.T) {
+	errorCodes := []int{404, 429, 500}
+	for _, code := range errorCodes {
+		testServer := httptest.NewServer(errorHandler(code))
+		defer testServer.Close()
+		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
+		require.NoError(t, err)
+		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
+		_, err = GetMSIToken(ctx, nil)
+		require.Error(t, err)
+		httpErr, ok := err.(httpError)
+		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
+		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
+	}
+}

backend/azurefiles/azurefiles_internal_test.go

@@ -1,70 +0,0 @@
-//go:build !plan9 && !js
-// +build !plan9,!js
-
-package azurefiles
-
-import (
-	"context"
-	"math/rand"
-	"strings"
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/stretchr/testify/assert"
-)
-
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Authentication", f.InternalTestAuth)
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)
-
-func (f *Fs) InternalTestAuth(t *testing.T) {
-	t.Skip("skipping since this requires authentication credentials which are not part of repo")
-	shareName := "test-rclone-oct-2023"
-	testCases := []struct {
-		name    string
-		options *Options
-	}{
-		{
-			name: "ConnectionString",
-			options: &Options{
-				ShareName:        shareName,
-				ConnectionString: "",
-			},
-		},
-		{
-			name: "AccountAndKey",
-			options: &Options{
-				ShareName: shareName,
-				Account:   "",
-				Key:       "",
-			}},
-		{
-			name: "SASUrl",
-			options: &Options{
-				ShareName: shareName,
-				SASURL:    "",
-			}},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			fs, err := newFsFromOptions(context.TODO(), "TestAzureFiles", "", tc.options)
-			assert.NoError(t, err)
-			dirName := randomString(10)
-			assert.NoError(t, fs.Mkdir(context.TODO(), dirName))
-		})
-	}
-}
-
-const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
-
-func randomString(charCount int) string {
-	strBldr := strings.Builder{}
-	for i := 0; i < charCount; i++ {
-		randPos := rand.Int63n(52)
-		strBldr.WriteByte(chars[randPos])
-	}
-	return strBldr.String()
-}

backend/azurefiles/azurefiles_test.go

@@ -1,18 +0,0 @@
-//go:build !plan9 && !js
-// +build !plan9,!js
-
-package azurefiles
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-func TestIntegration(t *testing.T) {
-	var objPtr *Object
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestAzureFiles:",
-		NilObject:  objPtr,
-	})
-}

backend/azurefiles/azurefiles_unsupported.go

@@ -1,7 +0,0 @@
-// Build for azurefiles for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-//go:build plan9 || js
-// +build plan9 js
-
-package azurefiles

backend/b2/api/types.go

@@ -37,14 +37,6 @@ type Bucket struct {
 	AccountID string `json:"accountId"`
 	Name      string `json:"bucketName"`
 	Type      string `json:"bucketType"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
-}
-
-// LifecycleRule is a single lifecycle rule
-type LifecycleRule struct {
-	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
-	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
-	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base
@@ -217,7 +209,6 @@ type CreateBucketRequest struct {
 	AccountID string `json:"accountId"`
 	Name      string `json:"bucketName"`
 	Type      string `json:"bucketType"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
 }
 
 // DeleteBucketRequest is used to create a bucket
@@ -340,11 +331,3 @@ type CopyPartRequest struct {
 	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
 	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
 }
-
-// UpdateBucketRequest describes a request to modify a B2 bucket
-type UpdateBucketRequest struct {
-	ID             string          `json:"bucketId"`
-	AccountID      string          `json:"accountId"`
-	Type           string          `json:"bucketType,omitempty"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
-}

backend/b2/b2.go

@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha1"
-	"encoding/json"
 	"errors"
 	"fmt"
 	gohash "hash"
@@ -33,7 +32,6 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
@@ -60,6 +58,8 @@ const (
 	defaultChunkSize    = 96 * fs.Mebi
 	defaultUploadCutoff = 200 * fs.Mebi
 	largeFileCopyCutoff = 4 * fs.Gibi              // 5E9 is the max
+	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap   = false
 )
 
 // Globals
@@ -74,17 +74,14 @@ func init() {
 		Name:        "b2",
 		Description: "Backblaze B2",
 		NewFs:       NewFs,
-		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "account",
 			Help:     "Account ID or Application Key ID.",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:     "key",
 			Help:     "Application Key.",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -150,18 +147,6 @@ might a maximum of "--transfers" chunks in progress at once.
 5,000,000 Bytes is the minimum size.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
-		}, {
-			Name: "upload_concurrency",
-			Help: `Concurrency for multipart uploads.
-
-This is the number of chunks of the same file that are uploaded
-concurrently.
-
-Note that chunks are stored in memory and there may be up to
-"--transfers" * "--b2-upload-concurrency" chunks stored at once
-in memory.`,
-			Default:  4,
-			Advanced: true,
 		}, {
 			Name: "disable_checksum",
 			Help: `Disable checksums for large (> upload cutoff) files.
@@ -201,41 +186,16 @@ The minimum value is 1 second. The maximum value is one week.`,
 			Advanced: true,
 		}, {
 			Name:     "memory_pool_flush_time",
-			Default:  fs.Duration(time.Minute),
+			Default:  memoryPoolFlushTime,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
-			Help:     `How often internal memory buffer pools will be flushed. (no longer used)`,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
 		}, {
 			Name:     "memory_pool_use_mmap",
-			Default:  false,
-			Advanced: true,
-			Hide:     fs.OptionHideBoth,
-			Help:     `Whether to use mmap buffers in internal memory pool. (no longer used)`,
-		}, {
-			Name: "lifecycle",
-			Help: `Set the number of days deleted files should be kept when creating a bucket.
-
-On bucket creation, this parameter is used to create a lifecycle rule
-for the entire bucket.
-
-If lifecycle is 0 (the default) it does not create a lifecycle rule so
-the default B2 behaviour applies. This is to create versions of files
-on delete and overwrite and to keep them indefinitely.
-
-If lifecycle is >0 then it creates a single rule setting the number of
-days before a file that is deleted or overwritten is deleted
-permanently. This is known as daysFromHidingToDeleting in the b2 docs.
-
-The minimum value for this parameter is 1 day.
-
-You can also enable hard_delete in the config also which will mean
-deletions won't cause versions but overwrites will still cause
-versions to be made.
-
-See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket creation.
-`,
-			Default:  0,
+			Default:  memoryPoolUseMmap,
 			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -262,11 +222,11 @@ type Options struct {
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
-	UploadConcurrency             int                  `config:"upload_concurrency"`
 	DisableCheckSum               bool                 `config:"disable_checksum"`
 	DownloadURL                   string               `config:"download_url"`
 	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
-	Lifecycle                     int                  `config:"lifecycle"`
+	MemoryPoolFlushTime           fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap             bool                 `config:"memory_pool_use_mmap"`
 	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -291,6 +251,7 @@ type Fs struct {
 	authMu          sync.Mutex                             // lock for authorizing the account
 	pacer           *fs.Pacer                              // To pace and retry the API calls
 	uploadToken     *pacer.TokenDispenser                  // control concurrency
+	pool            *pool.Pool                             // memory pool
 }
 
 // Object describes a b2 object
@@ -400,18 +361,11 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
-	body, err := rest.ReadBody(resp)
-	if err != nil {
-		fs.Errorf(nil, "Couldn't read error out of body: %v", err)
-		body = nil
-	}
-	// Decode error response if there was one - they can be blank
+	// Decode error response
 	errResponse := new(api.Error)
-	if len(body) > 0 {
-		err = json.Unmarshal(body, errResponse)
+	err := rest.DecodeJSON(resp, &errResponse)
 	if err != nil {
-			fs.Errorf(nil, "Couldn't decode error response: %v", err)
-		}
+		fs.Debugf(nil, "Couldn't decode error response: %v", err)
 	}
 	if errResponse.Code == "" {
 		errResponse.Code = "unknown"
@@ -455,14 +409,6 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-func (f *Fs) setCopyCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.CopyCutoff = f.opt.CopyCutoff, cs
-	}
-	return
-}
-
 // setRoot changes the root of the Fs
 func (f *Fs) setRoot(root string) {
 	f.root = parsePath(root)
@@ -510,6 +456,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		uploads:     make(map[string][]*api.GetUploadURLResponse),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			ci.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -517,7 +469,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		WriteMimeType:     true,
 		BucketBased:       true,
 		BucketBasedRootOK: true,
-		ChunkWriterDoesntSeek: true,
 	}).Fill(ctx, f)
 	// Set the test flag if required
 	if opt.TestMode != "" {
@@ -644,24 +595,23 @@ func (f *Fs) clearUploadURL(bucketID string) {
 	f.uploadMu.Unlock()
 }
 
-// getRW gets a RW buffer and an upload token
+// getBuf gets a buffer of f.opt.ChunkSize and an upload token
 //
 // If noBuf is set then it just gets an upload token
-func (f *Fs) getRW(noBuf bool) (rw *pool.RW) {
+func (f *Fs) getBuf(noBuf bool) (buf []byte) {
 	f.uploadToken.Get()
 	if !noBuf {
-		rw = multipart.NewRW()
+		buf = f.pool.Get()
 	}
-	return rw
+	return buf
 }
 
-// putRW returns a RW buffer to the memory pool and returns an upload
-// token
+// putBuf returns a buffer to the memory pool and an upload token
 //
-// If buf is nil then it just returns the upload token
-func (f *Fs) putRW(rw *pool.RW) {
-	if rw != nil {
-		_ = rw.Close()
+// If noBuf is set then it just returns the upload token
+func (f *Fs) putBuf(buf []byte, noBuf bool) {
+	if !noBuf {
+		f.pool.Put(buf)
 	}
 	f.uploadToken.Put()
 }
@@ -868,7 +818,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
 
 // listBuckets returns all the buckets to out
 func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
-	err = f.listBucketsToFn(ctx, "", func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		d := fs.NewDir(bucket.Name, time.Time{})
 		entries = append(entries, d)
 		return nil
@@ -961,14 +911,11 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 type listBucketFn func(*api.Bucket) error
 
 // listBucketsToFn lists the buckets to the function supplied
-func (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {
+func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error {
 	var account = api.ListBucketsRequest{
 		AccountID: f.info.AccountID,
 		BucketID:  f.info.Allowed.BucketID,
 	}
-	if bucketName != "" && account.BucketID == "" {
-		account.BucketName = f.opt.Enc.FromStandardName(bucketName)
-	}
 
 	var response api.ListBucketsResponse
 	opts := rest.Opts{
@@ -1014,7 +961,7 @@ func (f *Fs) getbucketType(ctx context.Context, bucket string) (bucketType strin
 	if bucketType != "" {
 		return bucketType, nil
 	}
-	err = f.listBucketsToFn(ctx, bucket, func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		// listBucketsToFn reads bucket Types
 		return nil
 	})
@@ -1049,7 +996,7 @@ func (f *Fs) getBucketID(ctx context.Context, bucket string) (bucketID string, e
 	if bucketID != "" {
 		return bucketID, nil
 	}
-	err = f.listBucketsToFn(ctx, bucket, func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		// listBucketsToFn sets IDs
 		return nil
 	})
@@ -1113,11 +1060,6 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 			Name:      f.opt.Enc.FromStandardName(bucket),
 			Type:      "allPrivate",
 		}
-		if f.opt.Lifecycle > 0 {
-			request.LifecycleRules = []api.LifecycleRule{{
-				DaysFromHidingToDeleting: &f.opt.Lifecycle,
-			}}
-		}
 		var response api.Bucket
 		err := f.pacer.Call(func() (bool, error) {
 			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
@@ -1279,7 +1221,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 					fs.Errorf(object.Name, "Can't create object %v", err)
 					continue
 				}
-				tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
+				tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
 				err = f.deleteByID(ctx, object.ID, object.Name)
 				checkErr(err)
 				tr.Done(ctx, err)
@@ -1293,7 +1235,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			if err != nil {
 				fs.Errorf(object, "Can't create object %+v", err)
 			}
-			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
+			tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
 			if oldOnly && last != remote {
 				// Check current version of the file
 				if object.Action == "hide" {
@@ -1338,7 +1280,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 // If newInfo is nil then the metadata will be copied otherwise it
 // will be replaced with newInfo
 func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
-	if srcObj.size > int64(f.opt.CopyCutoff) {
+	if srcObj.size >= int64(f.opt.CopyCutoff) {
 		if newInfo == nil {
 			newInfo, err = srcObj.getMetaData(ctx)
 			if err != nil {
@@ -1349,11 +1291,7 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
 		if err != nil {
 			return err
 		}
-		err = up.Copy(ctx)
-		if err != nil {
-			return err
-		}
-		return dstObj.decodeMetaDataFileInfo(up.info)
+		return up.Upload(ctx)
 	}
 
 	dstBucket, dstPath := dstObj.split()
@@ -1482,7 +1420,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	if err != nil {
 		return "", err
 	}
-	absPath := "/" + urlEncode(bucketPath)
+	absPath := "/" + bucketPath
 	link = RootURL + "/file/" + urlEncode(bucket) + absPath
 	bucketType, err := f.getbucketType(ctx, bucket)
 	if err != nil {
@@ -1921,11 +1859,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	if size < 0 {
+	if size == -1 {
 		// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
-		rw := o.fs.getRW(false)
+		buf := o.fs.getBuf(false)
 
-		n, err := io.CopyN(rw, in, int64(o.fs.opt.ChunkSize))
+		n, err := io.ReadFull(in, buf)
 		if err == nil {
 			bufReader := bufio.NewReader(in)
 			in = bufReader
@@ -1936,34 +1874,26 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			fs.Debugf(o, "File is big enough for chunked streaming")
 			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
-				o.fs.putRW(rw)
+				o.fs.putBuf(buf, false)
 				return err
 			}
 			// NB Stream returns the buffer and token
-			err = up.Stream(ctx, rw)
-			if err != nil {
-				return err
-			}
-			return o.decodeMetaDataFileInfo(up.info)
-		} else if err == io.EOF {
+			return up.Stream(ctx, buf)
+		} else if err == io.EOF || err == io.ErrUnexpectedEOF {
 			fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
-			defer o.fs.putRW(rw)
-			size = n
-			in = rw
+			defer o.fs.putBuf(buf, false)
+			size = int64(n)
+			in = bytes.NewReader(buf[:n])
 		} else {
-			o.fs.putRW(rw)
+			o.fs.putBuf(buf, false)
 			return err
 		}
 	} else if size > int64(o.fs.opt.UploadCutoff) {
-		chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
-			Open:        o.fs,
-			OpenOptions: options,
-		})
+		up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 		if err != nil {
 			return err
 		}
-		up := chunkWriter.(*largeUpload)
-		return o.decodeMetaDataFileInfo(up.info)
+		return up.Upload(ctx)
 	}
 
 	modTime := src.ModTime(ctx)
@@ -2071,41 +2001,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.decodeMetaDataFileInfo(&response)
 }
 
-// OpenChunkWriter returns the chunk size and a ChunkWriter
-//
-// Pass in the remote and the src object
-// You can also use options to hint at the desired chunk size
-func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
-	// FIXME what if file is smaller than 1 chunk?
-	if f.opt.Versions {
-		return info, nil, errNotWithVersions
-	}
-	if f.opt.VersionAt.IsSet() {
-		return info, nil, errNotWithVersionAt
-	}
-	//size := src.Size()
-
-	// Temporary Object under construction
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-
-	bucket, _ := o.split()
-	err = f.makeBucket(ctx, bucket)
-	if err != nil {
-		return info, nil, err
-	}
-
-	info = fs.ChunkWriterInfo{
-		ChunkSize:   int64(f.opt.ChunkSize),
-		Concurrency: o.fs.opt.UploadConcurrency,
-		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
-	}
-	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil)
-	return info, up, err
-}
-
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	bucket, bucketPath := o.split()
@@ -2131,137 +2026,6 @@ func (o *Object) ID() string {
 	return o.id
 }
 
-var lifecycleHelp = fs.CommandHelp{
-	Name:  "lifecycle",
-	Short: "Read or set the lifecycle for a bucket",
-	Long: `This command can be used to read or set the lifecycle for a bucket.
-
-Usage Examples:
-
-To show the current lifecycle rules:
-
-    rclone backend lifecycle b2:bucket
-
-This will dump something like this showing the lifecycle rules.
-
-    [
-        {
-            "daysFromHidingToDeleting": 1,
-            "daysFromUploadingToHiding": null,
-            "fileNamePrefix": ""
-        }
-    ]
-
-If there are no lifecycle rules (the default) then it will just return [].
-
-To reset the current lifecycle rules:
-
-    rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
-    rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
-
-This will run and then print the new lifecycle rules as above.
-
-Rclone only lets you set lifecycles for the whole bucket with the
-fileNamePrefix = "".
-
-You can't disable versioning with B2. The best you can do is to set
-the daysFromHidingToDeleting to 1 day. You can enable hard_delete in
-the config also which will mean deletions won't cause versions but
-overwrites will still cause versions to be made.
-
-    rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
-
-See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
-`,
-	Opts: map[string]string{
-		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
-		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
-	},
-}
-
-func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	var newRule api.LifecycleRule
-	if daysStr := opt["daysFromHidingToDeleting"]; daysStr != "" {
-		days, err := strconv.Atoi(daysStr)
-		if err != nil {
-			return nil, fmt.Errorf("bad daysFromHidingToDeleting: %w", err)
-		}
-		newRule.DaysFromHidingToDeleting = &days
-	}
-	if daysStr := opt["daysFromUploadingToHiding"]; daysStr != "" {
-		days, err := strconv.Atoi(daysStr)
-		if err != nil {
-			return nil, fmt.Errorf("bad daysFromUploadingToHiding: %w", err)
-		}
-		newRule.DaysFromUploadingToHiding = &days
-	}
-	bucketName, _ := f.split("")
-	if bucketName == "" {
-		return nil, errors.New("bucket required")
-
-	}
-
-	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
-		bucketID, err := f.getBucketID(ctx, bucketName)
-		if err != nil {
-			return nil, err
-		}
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/b2_update_bucket",
-		}
-		var request = api.UpdateBucketRequest{
-			ID:             bucketID,
-			AccountID:      f.info.AccountID,
-			LifecycleRules: []api.LifecycleRule{newRule},
-		}
-		var response api.Bucket
-		err = f.pacer.Call(func() (bool, error) {
-			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
-			return f.shouldRetry(ctx, resp, err)
-		})
-		if err != nil {
-			return nil, err
-		}
-		bucket = &response
-	} else {
-		err = f.listBucketsToFn(ctx, bucketName, func(b *api.Bucket) error {
-			bucket = b
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
-	}
-	if bucket == nil {
-		return nil, fs.ErrorDirNotFound
-	}
-	return bucket.LifecycleRules, nil
-}
-
-var commandHelp = []fs.CommandHelp{
-	lifecycleHelp,
-}
-
-// Command the backend to run a named command
-//
-// The command run is name
-// args may be used to read arguments from
-// opts may be used to read optional arguments from
-//
-// The result should be capable of being JSON encoded
-// If it is a string or a []string it will be shown to the user
-// otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	switch name {
-	case "lifecycle":
-		return f.lifecycleCommand(ctx, name, arg, opt)
-	default:
-		return nil, fs.ErrorCommandNotFound
-	}
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs           = &Fs{}
@@ -2271,8 +2035,6 @@ var (
 	_ fs.CleanUpper   = &Fs{}
 	_ fs.ListRer      = &Fs{}
 	_ fs.PublicLinker = &Fs{}
-	_ fs.OpenChunkWriter = &Fs{}
-	_ fs.Commander       = &Fs{}
 	_ fs.Object       = &Object{}
 	_ fs.MimeTyper    = &Object{}
 	_ fs.IDer         = &Object{}

backend/b2/b2_internal_test.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // Test b2 string encoding
@@ -169,10 +168,3 @@ func TestParseTimeString(t *testing.T) {
 	}
 
 }
-
-// -run TestIntegration/FsMkdir/FsPutFiles/Internal
-func (f *Fs) InternalTest(t *testing.T) {
-	// Internal tests go here
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/b2_test.go

@@ -28,12 +28,7 @@ func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadCutoff(cs)
 }
 
-func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setCopyCutoff(cs)
-}
-
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )

backend/b2/upload.go

@@ -5,6 +5,7 @@
 package b2
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha1"
 	"encoding/hex"
@@ -20,7 +21,6 @@ import (
 	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/sync/errgroup"
 )
@@ -78,14 +78,12 @@ type largeUpload struct {
 	wrap      accounting.WrapFn               // account parts being transferred
 	id        string                          // ID of the file being uploaded
 	size      int64                           // total size
-	parts     int                             // calculated number of parts, if known
-	sha1smu   sync.Mutex                      // mutex to protect sha1s
+	parts     int64                           // calculated number of parts, if known
 	sha1s     []string                        // slice of SHA1s for each part
 	uploadMu  sync.Mutex                      // lock for upload variable
 	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
 	chunkSize int64                           // chunk size to use
 	src       *Object                         // if copying, object we are reading from
-	info      *api.FileInfo                   // final response with info about the object
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
@@ -93,16 +91,18 @@ type largeUpload struct {
 // If newInfo is set then metadata from that will be used instead of reading it from src
 func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
-	parts := 0
+	parts := int64(0)
+	sha1SliceSize := int64(maxParts)
 	chunkSize := defaultChunkSize
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
 		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
-		parts = int(size / int64(chunkSize))
+		parts = size / int64(chunkSize)
 		if size%int64(chunkSize) != 0 {
 			parts++
 		}
+		sha1SliceSize = parts
 	}
 
 	opts := rest.Opts{
@@ -150,7 +150,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		id:        response.ID,
 		size:      size,
 		parts:     parts,
-		sha1s:     make([]string, 0, 16),
+		sha1s:     make([]string, sha1SliceSize),
 		chunkSize: int64(chunkSize),
 	}
 	// unwrap the accounting from the input, we use wrap to put it
@@ -169,13 +169,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 // This should be returned with returnUploadURL when finished
 func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
 	up.uploadMu.Lock()
-	if len(up.uploads) > 0 {
-		upload, up.uploads = up.uploads[0], up.uploads[1:]
-		up.uploadMu.Unlock()
-		return upload, nil
-	}
-	up.uploadMu.Unlock()
-
+	defer up.uploadMu.Unlock()
+	if len(up.uploads) == 0 {
 		opts := rest.Opts{
 			Method: "POST",
 			Path:   "/b2_get_upload_part_url",
@@ -183,13 +178,16 @@ func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadP
 		var request = api.GetUploadPartURLRequest{
 			ID: up.id,
 		}
-	err = up.f.pacer.Call(func() (bool, error) {
+		err := up.f.pacer.Call(func() (bool, error) {
 			resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
 			return up.f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to get upload URL: %w", err)
 		}
+	} else {
+		upload, up.uploads = up.uploads[0], up.uploads[1:]
+	}
 	return upload, nil
 }
 
@@ -203,39 +201,10 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
-// Add an sha1 to the being built up sha1s
-func (up *largeUpload) addSha1(chunkNumber int, sha1 string) {
-	up.sha1smu.Lock()
-	defer up.sha1smu.Unlock()
-	if len(up.sha1s) < chunkNumber+1 {
-		up.sha1s = append(up.sha1s, make([]string, chunkNumber+1-len(up.sha1s))...)
-	}
-	up.sha1s[chunkNumber] = sha1
-}
-
-// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
-func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (size int64, err error) {
-	// Only account after the checksum reads have been done
-	if do, ok := reader.(pool.DelayAccountinger); ok {
-		// To figure out this number, do a transfer and if the accounted size is 0 or a
-		// multiple of what it should be, increase or decrease this number.
-		do.DelayAccounting(1)
-	}
-
-	err = up.f.pacer.Call(func() (bool, error) {
-		// Discover the size by seeking to the end
-		size, err = reader.Seek(0, io.SeekEnd)
-		if err != nil {
-			return false, err
-		}
-
-		// rewind the reader on retry and after reading size
-		_, err = reader.Seek(0, io.SeekStart)
-		if err != nil {
-			return false, err
-		}
-
-		fs.Debugf(up.o, "Sending chunk %d length %d", chunkNumber, size)
+// Transfer a chunk
+func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
 
 		// Get upload URL
 		upload, err := up.getUploadURL(ctx)
@@ -243,8 +212,8 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 			return false, err
 		}
 
-		in := newHashAppendingReader(reader, sha1.New())
-		sizeWithHash := size + int64(in.AdditionalLength())
+		in := newHashAppendingReader(bytes.NewReader(body), sha1.New())
+		size := int64(len(body)) + int64(in.AdditionalLength())
 
 		// Authorization
 		//
@@ -274,10 +243,10 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 			Body:    up.wrap(in),
 			ExtraHeaders: map[string]string{
 				"Authorization":    upload.AuthorizationToken,
-				"X-Bz-Part-Number": fmt.Sprintf("%d", chunkNumber+1),
+				"X-Bz-Part-Number": fmt.Sprintf("%d", part),
 				sha1Header:         "hex_digits_at_end",
 			},
-			ContentLength: &sizeWithHash,
+			ContentLength: &size,
 		}
 
 		var response api.UploadPartResponse
@@ -285,7 +254,7 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 		resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
 		retry, err := up.f.shouldRetry(ctx, resp, err)
 		if err != nil {
-			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", chunkNumber, retry, err, err)
+			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
 		// On retryable error clear PartUploadURL
 		if retry {
@@ -293,30 +262,30 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 			upload = nil
 		}
 		up.returnUploadURL(upload)
-		up.addSha1(chunkNumber, in.HexSum())
+		up.sha1s[part-1] = in.HexSum()
 		return retry, err
 	})
 	if err != nil {
-		fs.Debugf(up.o, "Error sending chunk %d: %v", chunkNumber, err)
+		fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
 	} else {
-		fs.Debugf(up.o, "Done sending chunk %d", chunkNumber)
+		fs.Debugf(up.o, "Done sending chunk %d", part)
 	}
-	return size, err
+	return err
 }
 
 // Copy a chunk
-func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64) error {
+func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
 		opts := rest.Opts{
 			Method: "POST",
 			Path:   "/b2_copy_part",
 		}
-		offset := int64(part) * up.chunkSize // where we are in the source file
+		offset := (part - 1) * up.chunkSize // where we are in the source file
 		var request = api.CopyPartRequest{
 			SourceID:    up.src.id,
 			LargeFileID: up.id,
-			PartNumber:  int64(part + 1),
+			PartNumber:  part,
 			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
 		}
 		var response api.UploadPartResponse
@@ -325,7 +294,7 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64)
 		if err != nil {
 			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
-		up.addSha1(part, response.SHA1)
+		up.sha1s[part-1] = response.SHA1
 		return retry, err
 	})
 	if err != nil {
@@ -336,8 +305,8 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64)
 	return err
 }
 
-// Close closes off the large upload
-func (up *largeUpload) Close(ctx context.Context) error {
+// finish closes off the large upload
+func (up *largeUpload) finish(ctx context.Context) error {
 	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
 	opts := rest.Opts{
 		Method: "POST",
@@ -355,12 +324,11 @@ func (up *largeUpload) Close(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	up.info = &response
-	return nil
+	return up.o.decodeMetaDataFileInfo(&response)
 }
 
-// Abort aborts the large upload
-func (up *largeUpload) Abort(ctx context.Context) error {
+// cancel aborts the large upload
+func (up *largeUpload) cancel(ctx context.Context) error {
 	fs.Debugf(up.o, "Cancelling large file %s", up.what)
 	opts := rest.Opts{
 		Method: "POST",
@@ -385,89 +353,94 @@ func (up *largeUpload) Abort(ctx context.Context) error {
 // reaches EOF.
 //
 // Note that initialUploadBlock must be returned to f.putBuf()
-func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock *pool.RW) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
+func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
 	var (
 		g, gCtx      = errgroup.WithContext(ctx)
 		hasMoreParts = true
 	)
-	up.size = initialUploadBlock.Size()
-	up.parts = 0
-	for part := 0; hasMoreParts; part++ {
+	up.size = int64(len(initialUploadBlock))
+	g.Go(func() error {
+		for part := int64(1); hasMoreParts; part++ {
 			// Get a block of memory from the pool and token which limits concurrency.
-		var rw *pool.RW
-		if part == 0 {
-			rw = initialUploadBlock
+			var buf []byte
+			if part == 1 {
+				buf = initialUploadBlock
 			} else {
-			rw = up.f.getRW(false)
+				buf = up.f.getBuf(false)
 			}
 
 			// Fail fast, in case an errgroup managed function returns an error
 			// gCtx is cancelled. There is no point in uploading all the other parts.
 			if gCtx.Err() != nil {
-			up.f.putRW(rw)
-			break
+				up.f.putBuf(buf, false)
+				return nil
 			}
 
 			// Read the chunk
-		var n int64
-		if part == 0 {
-			n = rw.Size()
+			var n int
+			if part == 1 {
+				n = len(buf)
 			} else {
-			n, err = io.CopyN(rw, up.in, up.chunkSize)
-			if err == io.EOF {
-				if n == 0 {
-					fs.Debugf(up.o, "Not sending empty chunk after EOF - ending.")
-					up.f.putRW(rw)
-					break
-				} else {
-					fs.Debugf(up.o, "Read less than a full chunk %d, making this the last one.", n)
-				}
+				n, err = io.ReadFull(up.in, buf)
+				if err == io.ErrUnexpectedEOF {
+					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+					buf = buf[:n]
 					hasMoreParts = false
+				} else if err == io.EOF {
+					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+					up.f.putBuf(buf, false)
+					return nil
 				} else if err != nil {
 					// other kinds of errors indicate failure
-				up.f.putRW(rw)
+					up.f.putBuf(buf, false)
 					return err
 				}
 			}
 
 			// Keep stats up to date
-		up.parts += 1
-		up.size += n
+			up.parts = part
+			up.size += int64(n)
 			if part > maxParts {
-			up.f.putRW(rw)
+				up.f.putBuf(buf, false)
 				return fmt.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
 			}
 
 			part := part // for the closure
 			g.Go(func() (err error) {
-			defer up.f.putRW(rw)
-			_, err = up.WriteChunk(gCtx, part, rw)
-			return err
+				defer up.f.putBuf(buf, false)
+				return up.transferChunk(gCtx, part, buf)
 			})
 		}
+		return nil
+	})
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
-	return up.Close(ctx)
+	up.sha1s = up.sha1s[:up.parts]
+	return up.finish(ctx)
 }
 
-// Copy the chunks from the source to the destination
-func (up *largeUpload) Copy(ctx context.Context) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
+// Upload uploads the chunks from the input
+func (up *largeUpload) Upload(ctx context.Context) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
 	var (
 		g, gCtx   = errgroup.WithContext(ctx)
 		remaining = up.size
 	)
-	g.SetLimit(up.f.opt.UploadConcurrency)
-	for part := 0; part < up.parts; part++ {
+	g.Go(func() error {
+		for part := int64(1); part <= up.parts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			buf := up.f.getBuf(up.doCopy)
+
 			// Fail fast, in case an errgroup managed function returns an error
-		// gCtx is cancelled. There is no point in copying all the other parts.
+			// gCtx is cancelled. There is no point in uploading all the other parts.
 			if gCtx.Err() != nil {
-			break
+				up.f.putBuf(buf, up.doCopy)
+				return nil
 			}
 
 			reqSize := remaining
@@ -475,15 +448,33 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
 				reqSize = up.chunkSize
 			}
 
+			if !up.doCopy {
+				// Read the chunk
+				buf = buf[:reqSize]
+				_, err = io.ReadFull(up.in, buf)
+				if err != nil {
+					up.f.putBuf(buf, up.doCopy)
+					return err
+				}
+			}
+
 			part := part // for the closure
 			g.Go(func() (err error) {
-			return up.copyChunk(gCtx, part, reqSize)
+				defer up.f.putBuf(buf, up.doCopy)
+				if !up.doCopy {
+					err = up.transferChunk(gCtx, part, buf)
+				} else {
+					err = up.copyChunk(gCtx, part, reqSize)
+				}
+				return err
 			})
 			remaining -= reqSize
 		}
+		return nil
+	})
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
-	return up.Close(ctx)
+	return up.finish(ctx)
 }

backend/box/api/types.go

@@ -52,7 +52,7 @@ func (e *Error) Error() string {
 		out += ": " + e.Message
 	}
 	if e.ContextInfo != nil {
-		out += fmt.Sprintf(" (%s)", string(e.ContextInfo))
+		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
 	}
 	return out
 }
@@ -63,7 +63,7 @@ var _ error = (*Error)(nil)
 // ItemFields are the fields needed for FileInfo
 var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link,owned_by"
 
-// Types of things in Item/ItemMini
+// Types of things in Item
 const (
 	ItemTypeFolder    = "folder"
 	ItemTypeFile      = "file"
@@ -72,21 +72,11 @@ const (
 	ItemStatusDeleted = "deleted"
 )
 
-// ItemMini is a subset of the elements in a full Item returned by some API calls
-type ItemMini struct {
-	Type       string `json:"type"`
-	ID         string `json:"id"`
-	SequenceID int64  `json:"sequence_id,string"`
-	Etag       string `json:"etag"`
-	SHA1       string `json:"sha1"`
-	Name       string `json:"name"`
-}
-
 // Item describes a folder or a file as returned by Get Folder Items and others
 type Item struct {
 	Type              string  `json:"type"`
 	ID                string  `json:"id"`
-	SequenceID        int64    `json:"sequence_id,string"`
+	SequenceID        string  `json:"sequence_id"`
 	Etag              string  `json:"etag"`
 	SHA1              string  `json:"sha1"`
 	Name              string  `json:"name"`
@@ -96,7 +86,6 @@ type Item struct {
 	ContentCreatedAt  Time    `json:"content_created_at"`
 	ContentModifiedAt Time    `json:"content_modified_at"`
 	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
-	Parent            ItemMini `json:"parent"`
 	SharedLink        struct {
 		URL    string `json:"url,omitempty"`
 		Access string `json:"access,omitempty"`
@@ -167,7 +156,19 @@ type PreUploadCheckResponse struct {
 // PreUploadCheckConflict is returned in the ContextInfo error field
 // from PreUploadCheck when the error code is "item_name_in_use"
 type PreUploadCheckConflict struct {
-	Conflicts ItemMini `json:"conflicts"`
+	Conflicts struct {
+		Type        string `json:"type"`
+		ID          string `json:"id"`
+		FileVersion struct {
+			Type string `json:"type"`
+			ID   string `json:"id"`
+			Sha1 string `json:"sha1"`
+		} `json:"file_version"`
+		SequenceID string `json:"sequence_id"`
+		Etag       string `json:"etag"`
+		Sha1       string `json:"sha1"`
+		Name       string `json:"name"`
+	} `json:"conflicts"`
 }
 
 // UpdateFileModTime is used in Update File Info
@@ -280,30 +281,3 @@ type User struct {
 	Address       string    `json:"address"`
 	AvatarURL     string    `json:"avatar_url"`
 }
-
-// FileTreeChangeEventTypes are the events that can require cache invalidation
-var FileTreeChangeEventTypes = map[string]struct{}{
-	"ITEM_COPY":                 {},
-	"ITEM_CREATE":               {},
-	"ITEM_MAKE_CURRENT_VERSION": {},
-	"ITEM_MODIFY":               {},
-	"ITEM_MOVE":                 {},
-	"ITEM_RENAME":               {},
-	"ITEM_TRASH":                {},
-	"ITEM_UNDELETE_VIA_TRASH":   {},
-	"ITEM_UPLOAD":               {},
-}
-
-// Event is an array element in the response returned from /events
-type Event struct {
-	EventType string `json:"event_type"`
-	EventID   string `json:"event_id"`
-	Source    Item   `json:"source"`
-}
-
-// Events is returned from /events
-type Events struct {
-	ChunkSize          int64   `json:"chunk_size"`
-	Entries            []Event `json:"entries"`
-	NextStreamPosition int64   `json:"next_stream_position"`
-}

backend/box/box.go

@@ -27,7 +27,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/golang-jwt/jwt/v4"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -46,6 +45,7 @@ import (
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/jws"
 )
 
 const (
@@ -76,11 +76,6 @@ var (
 	}
 )
 
-type boxCustomClaims struct {
-	jwt.StandardClaims
-	BoxSubType string `json:"box_sub_type,omitempty"`
-}
-
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -111,14 +106,12 @@ func init() {
 			Help:     "Fill in for rclone to use a non root folder as its starting point.",
 			Default:  "0",
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "box_config_file",
 			Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp,
 		}, {
 			Name: "access_token",
 			Help: "Box App Primary Access Token\n\nLeave blank normally.",
-			Sensitive: true,
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -149,23 +142,6 @@ func init() {
 			Default:  "",
 			Help:     "Only show items owned by the login (email address) passed in.",
 			Advanced: true,
-		}, {
-			Name:    "impersonate",
-			Default: "",
-			Help: `Impersonate this user ID when using a service account.
-
-Setting this flag allows rclone, when using a JWT service account, to
-act on behalf of another user by setting the as-user header.
-
-The user ID is the Box identifier for a user. User IDs can found for
-any user via the GET /users endpoint, which is only available to
-admins, or by calling the GET /users/me endpoint with an authenticated
-user session.
-
-See: https://developer.box.com/guides/authentication/jwt/as-user/
-`,
-			Advanced:  true,
-			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -202,7 +178,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
 	client := fshttp.NewClient(ctx)
-	err = jwtutil.Config("box", name, tokenURL, *claims, signingHeaders, queryParams, privateKey, m, client)
+	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
 	return err
 }
 
@@ -218,31 +194,34 @@ func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
 	return boxConfig, nil
 }
 
-func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomClaims, err error) {
+func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) {
 	val, err := jwtutil.RandomHex(20)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to generate random string for jti: %w", err)
 	}
 
-	claims = &boxCustomClaims{
-		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
-		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
-		StandardClaims: jwt.StandardClaims{
-			Id:        val,
-			Issuer:    boxConfig.BoxAppSettings.ClientID,
-			Subject:   boxConfig.EnterpriseID,
-			Audience:  tokenURL,
-			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
+	claims = &jws.ClaimSet{
+		Iss: boxConfig.BoxAppSettings.ClientID,
+		Sub: boxConfig.EnterpriseID,
+		Aud: tokenURL,
+		Exp: time.Now().Add(time.Second * 45).Unix(),
+		PrivateClaims: map[string]interface{}{
+			"box_sub_type": boxSubType,
+			"aud":          tokenURL,
+			"jti":          val,
 		},
-		BoxSubType: boxSubType,
 	}
+
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
-	signingHeaders := map[string]interface{}{
-		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
+func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header {
+	signingHeaders := &jws.Header{
+		Algorithm: "RS256",
+		Typ:       "JWT",
+		KeyID:     boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
+
 	return signingHeaders
 }
 
@@ -279,14 +258,6 @@ type Options struct {
 	AccessToken   string               `config:"access_token"`
 	ListChunk     int                  `config:"list_chunk"`
 	OwnedBy       string               `config:"owned_by"`
-	Impersonate   string               `config:"impersonate"`
-}
-
-// ItemMeta defines metadata we cache for each Item ID
-type ItemMeta struct {
-	SequenceID int64  // the most recent event processed for this item
-	ParentID   string // ID of the parent directory of this item
-	Name       string // leaf name of this item
 }
 
 // Fs represents a remote box
@@ -300,8 +271,6 @@ type Fs struct {
 	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
-	itemMetaCacheMu *sync.Mutex           // protects itemMetaCache
-	itemMetaCache   map[string]ItemMeta   // map of Item ID to selected metadata
 }
 
 // Object describes a box object
@@ -380,7 +349,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
-	// defer log.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
+	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -389,30 +358,20 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		return nil, err
 	}
 
-	// Use preupload to find the ID
-	itemMini, err := f.preUploadCheck(ctx, leaf, directoryID, -1)
-	if err != nil {
-		return nil, err
+	found, err := f.listAll(ctx, directoryID, false, true, true, func(item *api.Item) bool {
+		if strings.EqualFold(item.Name, leaf) {
+			info = item
+			return true
 		}
-	if itemMini == nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-
-	// Now we have the ID we can look up the object proper
-	opts := rest.Opts{
-		Method:     "GET",
-		Path:       "/files/" + itemMini.ID,
-		Parameters: fieldsValue(),
-	}
-	var item api.Item
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, nil, &item)
-		return shouldRetry(ctx, resp, err)
+		return false
 	})
 	if err != nil {
 		return nil, err
 	}
-	return &item, nil
+	if !found {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return info, nil
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -465,8 +424,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		srv:         rest.NewClient(client).SetRoot(rootURL),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
-		itemMetaCacheMu: new(sync.Mutex),
-		itemMetaCache:   make(map[string]ItemMeta),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -479,11 +436,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken)
 	}
 
-	// If using impersonate set an as-user header
-	if f.opt.Impersonate != "" {
-		f.srv.SetHeader("as-user", f.opt.Impersonate)
-	}
-
 	jsonFile, ok := m.Get("box_config_file")
 	boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 
@@ -726,17 +678,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			entries = append(entries, o)
 		}
-
-		// Cache some metadata for this Item to help us process events later
-		// on. In particular, the box event API does not provide the old path
-		// of the Item when it is renamed/deleted/moved/etc.
-		f.itemMetaCacheMu.Lock()
-		cachedItemMeta, found := f.itemMetaCache[info.ID]
-		if !found || cachedItemMeta.SequenceID < info.SequenceID {
-			f.itemMetaCache[info.ID] = ItemMeta{SequenceID: info.SequenceID, ParentID: directoryID, Name: info.Name}
-		}
-		f.itemMetaCacheMu.Unlock()
-
 		return false
 	})
 	if err != nil {
@@ -772,7 +713,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 //
 // It returns "", nil if the file is good to go
 // It returns "ID", nil if the file must be updated
-func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (item *api.ItemMini, err error) {
+func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
 	check := api.PreUploadCheck{
 		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
@@ -797,16 +738,16 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size
 			var conflict api.PreUploadCheckConflict
 			err = json.Unmarshal(apiErr.ContextInfo, &conflict)
 			if err != nil {
-				return nil, fmt.Errorf("pre-upload check: JSON decode failed: %w", err)
+				return "", fmt.Errorf("pre-upload check: JSON decode failed: %w", err)
 			}
 			if conflict.Conflicts.Type != api.ItemTypeFile {
-				return nil, fs.ErrorIsDir
+				return "", fmt.Errorf("pre-upload check: can't overwrite non file with file: %w", err)
 			}
-			return &conflict.Conflicts, nil
+			return conflict.Conflicts.ID, nil
 		}
-		return nil, fmt.Errorf("pre-upload check: %w", err)
+		return "", fmt.Errorf("pre-upload check: %w", err)
 	}
-	return nil, nil
+	return "", nil
 }
 
 // Put the object
@@ -827,11 +768,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 	// Preflight check the upload, which returns the ID if the
 	// object already exists
-	item, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
+	ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
 	if err != nil {
 		return nil, err
 	}
-	if item == nil {
+	if ID == "" {
 		return f.PutUnchecked(ctx, in, src, options...)
 	}
 
@@ -839,7 +780,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	o := &Object{
 		fs:     f,
 		remote: remote,
-		id:     item.ID,
+		id:     ID,
 	}
 	return o, o.Update(ctx, in, src, options...)
 }
@@ -1176,7 +1117,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 // CleanUp empties the trash
 func (f *Fs) CleanUp(ctx context.Context) (err error) {
 	var (
-		deleteErrors       atomic.Uint64
+		deleteErrors       = int64(0)
 		concurrencyControl = make(chan struct{}, fs.GetConfig(ctx).Checkers)
 		wg                 sync.WaitGroup
 	)
@@ -1192,7 +1133,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 				err := f.deletePermanently(ctx, item.Type, item.ID)
 				if err != nil {
 					fs.Errorf(f, "failed to delete trash item %q (%q): %v", item.Name, item.ID, err)
-					deleteErrors.Add(1)
+					atomic.AddInt64(&deleteErrors, 1)
 				}
 			}()
 		} else {
@@ -1201,277 +1142,12 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		return false
 	})
 	wg.Wait()
-	if deleteErrors.Load() != 0 {
-		return fmt.Errorf("failed to delete %d trash items", deleteErrors.Load())
+	if deleteErrors != 0 {
+		return fmt.Errorf("failed to delete %d trash items", deleteErrors)
 	}
 	return err
 }
 
-// ChangeNotify calls the passed function with a path that has had changes.
-// If the implementation uses polling, it should adhere to the given interval.
-//
-// Automatically restarts itself in case of unexpected behavior of the remote.
-//
-// Close the returned channel to stop being notified.
-func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
-	go func() {
-		// get the `stream_position` early so all changes from now on get processed
-		streamPosition, err := f.changeNotifyStreamPosition(ctx)
-		if err != nil {
-			fs.Infof(f, "Failed to get StreamPosition: %s", err)
-		}
-
-		// box can send duplicate Event IDs. Use this map to track and filter
-		// the ones we've already processed.
-		processedEventIDs := make(map[string]time.Time)
-
-		var ticker *time.Ticker
-		var tickerC <-chan time.Time
-		for {
-			select {
-			case pollInterval, ok := <-pollIntervalChan:
-				if !ok {
-					if ticker != nil {
-						ticker.Stop()
-					}
-					return
-				}
-				if ticker != nil {
-					ticker.Stop()
-					ticker, tickerC = nil, nil
-				}
-				if pollInterval != 0 {
-					ticker = time.NewTicker(pollInterval)
-					tickerC = ticker.C
-				}
-			case <-tickerC:
-				if streamPosition == "" {
-					streamPosition, err = f.changeNotifyStreamPosition(ctx)
-					if err != nil {
-						fs.Infof(f, "Failed to get StreamPosition: %s", err)
-						continue
-					}
-				}
-
-				// Garbage collect EventIDs older than 1 minute
-				for eventID, timestamp := range processedEventIDs {
-					if time.Since(timestamp) > time.Minute {
-						delete(processedEventIDs, eventID)
-					}
-				}
-
-				streamPosition, err = f.changeNotifyRunner(ctx, notifyFunc, streamPosition, processedEventIDs)
-				if err != nil {
-					fs.Infof(f, "Change notify listener failure: %s", err)
-				}
-			}
-		}
-	}()
-}
-
-func (f *Fs) changeNotifyStreamPosition(ctx context.Context) (streamPosition string, err error) {
-	opts := rest.Opts{
-		Method:     "GET",
-		Path:       "/events",
-		Parameters: fieldsValue(),
-	}
-	opts.Parameters.Set("stream_position", "now")
-	opts.Parameters.Set("stream_type", "changes")
-
-	var result api.Events
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return "", err
-	}
-
-	return strconv.FormatInt(result.NextStreamPosition, 10), nil
-}
-
-// Attempts to construct the full path for an object, given the ID of its
-// parent directory and the name of the object.
-//
-// Can return "" if the parentID is not currently in the directory cache.
-func (f *Fs) getFullPath(parentID string, childName string) (fullPath string) {
-	fullPath = ""
-	name := f.opt.Enc.ToStandardName(childName)
-	if parentID != "" {
-		if parentDir, ok := f.dirCache.GetInv(parentID); ok {
-			if len(parentDir) > 0 {
-				fullPath = parentDir + "/" + name
-			} else {
-				fullPath = name
-			}
-		}
-	} else {
-		// No parent, this object is at the root
-		fullPath = name
-	}
-	return fullPath
-}
-
-func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), streamPosition string, processedEventIDs map[string]time.Time) (nextStreamPosition string, err error) {
-	nextStreamPosition = streamPosition
-
-	for {
-		limit := f.opt.ListChunk
-
-		// box only allows a max of 500 events
-		if limit > 500 {
-			limit = 500
-		}
-
-		opts := rest.Opts{
-			Method:     "GET",
-			Path:       "/events",
-			Parameters: fieldsValue(),
-		}
-		opts.Parameters.Set("stream_position", nextStreamPosition)
-		opts.Parameters.Set("stream_type", "changes")
-		opts.Parameters.Set("limit", strconv.Itoa(limit))
-
-		var result api.Events
-		var resp *http.Response
-		fs.Debugf(f, "Checking for changes on remote (next_stream_position: %q)", nextStreamPosition)
-		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
-		})
-		if err != nil {
-			return "", err
-		}
-
-		if result.ChunkSize != int64(len(result.Entries)) {
-			return "", fmt.Errorf("invalid response to event request, chunk_size (%v) not equal to number of entries (%v)", result.ChunkSize, len(result.Entries))
-		}
-
-		nextStreamPosition = strconv.FormatInt(result.NextStreamPosition, 10)
-		if result.ChunkSize == 0 {
-			return nextStreamPosition, nil
-		}
-
-		type pathToClear struct {
-			path      string
-			entryType fs.EntryType
-		}
-		var pathsToClear []pathToClear
-		newEventIDs := 0
-		for _, entry := range result.Entries {
-			eventDetails := fmt.Sprintf("[%q(%d)|%s|%s|%s|%s]", entry.Source.Name, entry.Source.SequenceID,
-				entry.Source.Type, entry.EventType, entry.Source.ID, entry.EventID)
-
-			if entry.EventID == "" {
-				fs.Debugf(f, "%s ignored due to missing EventID", eventDetails)
-				continue
-			}
-			if _, ok := processedEventIDs[entry.EventID]; ok {
-				fs.Debugf(f, "%s ignored due to duplicate EventID", eventDetails)
-				continue
-			}
-			processedEventIDs[entry.EventID] = time.Now()
-			newEventIDs++
-
-			if entry.Source.ID == "" { // missing File or Folder ID
-				fs.Debugf(f, "%s ignored due to missing SourceID", eventDetails)
-				continue
-			}
-			if entry.Source.Type != api.ItemTypeFile && entry.Source.Type != api.ItemTypeFolder { // event is not for a file or folder
-				fs.Debugf(f, "%s ignored due to unsupported SourceType", eventDetails)
-				continue
-			}
-
-			// Only interested in event types that result in a file tree change
-			if _, found := api.FileTreeChangeEventTypes[entry.EventType]; !found {
-				fs.Debugf(f, "%s ignored due to unsupported EventType", eventDetails)
-				continue
-			}
-
-			f.itemMetaCacheMu.Lock()
-			itemMeta, cachedItemMetaFound := f.itemMetaCache[entry.Source.ID]
-			if cachedItemMetaFound {
-				if itemMeta.SequenceID >= entry.Source.SequenceID {
-					// Item in the cache has the same or newer SequenceID than
-					// this event. Ignore this event, it must be old.
-					f.itemMetaCacheMu.Unlock()
-					fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
-					continue
-				}
-
-				// This event is newer. Delete its entry from the cache,
-				// we'll notify about its change below, then it's up to a
-				// future list operation to repopulate the cache.
-				delete(f.itemMetaCache, entry.Source.ID)
-			}
-			f.itemMetaCacheMu.Unlock()
-
-			entryType := fs.EntryDirectory
-			if entry.Source.Type == api.ItemTypeFile {
-				entryType = fs.EntryObject
-			}
-
-			// The box event only includes the new path for the object (e.g.
-			// the path after the object was moved). If there was an old path
-			// saved in our cache, it must be cleared.
-			if cachedItemMetaFound {
-				path := f.getFullPath(itemMeta.ParentID, itemMeta.Name)
-				if path != "" {
-					fs.Debugf(f, "%s added old path (%q) for notify", eventDetails, path)
-					pathsToClear = append(pathsToClear, pathToClear{path: path, entryType: entryType})
-				} else {
-					fs.Debugf(f, "%s old parent not cached", eventDetails)
-				}
-
-				// If this is a directory, also delete it from the dir cache.
-				// This will effectively invalidate the item metadata cache
-				// entries for all descendents of this directory, since we
-				// will no longer be able to construct a full path for them.
-				// This is exactly what we want, since we don't want to notify
-				// on the paths of these descendents if one of their ancestors
-				// has been renamed/deleted.
-				if entry.Source.Type == api.ItemTypeFolder {
-					f.dirCache.FlushDir(path)
-				}
-			}
-
-			// If the item is "active", then it is not trashed or deleted, so
-			// it potentially has a valid parent.
-			//
-			// Construct the new path of the object, based on the Parent ID
-			// and its name. If we get an empty result, it means we don't
-			// currently know about this object so notification is unnecessary.
-			if entry.Source.ItemStatus == api.ItemStatusActive {
-				path := f.getFullPath(entry.Source.Parent.ID, entry.Source.Name)
-				if path != "" {
-					fs.Debugf(f, "%s added new path (%q) for notify", eventDetails, path)
-					pathsToClear = append(pathsToClear, pathToClear{path: path, entryType: entryType})
-				} else {
-					fs.Debugf(f, "%s new parent not found", eventDetails)
-				}
-			}
-		}
-
-		// box can sometimes repeatedly return the same Event IDs within a
-		// short period of time. If it stops giving us new ones, treat it
-		// the same as if it returned us none at all.
-		if newEventIDs == 0 {
-			return nextStreamPosition, nil
-		}
-
-		notifiedPaths := make(map[string]bool)
-		for _, p := range pathsToClear {
-			if _, ok := notifiedPaths[p.path]; ok {
-				continue
-			}
-			notifiedPaths[p.path] = true
-			notifyFunc(p.path, p.entryType)
-		}
-		fs.Debugf(f, "Received %v events, resulting in %v paths and %v notifications", len(result.Entries), len(pathsToClear), len(notifiedPaths))
-	}
-}
-
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {

backend/cache/cache.go

@@ -78,7 +78,6 @@ func init() {
 		}, {
 			Name: "plex_username",
 			Help: "The username of the Plex user.",
-			Sensitive: true,
 		}, {
 			Name:       "plex_password",
 			Help:       "The password of the Plex user.",
@@ -88,7 +87,6 @@ func init() {
 			Help:     "The plex token for authentication - auto set normally.",
 			Hide:     fs.OptionHideBoth,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name:     "plex_insecure",
 			Help:     "Skip all certificate verification when connecting to the Plex server.",
@@ -1040,7 +1038,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		}
 		fs.Debugf(dir, "list: remove entry: %v", entryRemote)
 	}
-	entries = nil //nolint:ineffassign
+	entries = nil
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
 	var batchDirectories []*Directory
@@ -1789,7 +1787,7 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
 	}
 }
 
-// StopBackgroundRunners will signal all the runners to stop their work
+// StopBackgroundRunners will signall all the runners to stop their work
 // can be triggered from a terminate signal or from testing between runs
 func (f *Fs) StopBackgroundRunners() {
 	f.cleanupChan <- false

backend/cache/cache_internal_test.go

@@ -101,12 +101,14 @@ func TestMain(m *testing.M) {
 
 func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	id := fmt.Sprintf("tilrair%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	// Instantiate inner fs
 	innerFolder := "inner"
 	runInstance.mkdir(t, rootFs, innerFolder)
-	rootFs2, _ := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil)
+	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs2, boltDb2)
 
 	runInstance.writeObjectString(t, rootFs2, "one", "content")
 	listRoot, err := runInstance.list(t, rootFs, "")
@@ -223,7 +225,8 @@ func TestInternalVfsCache(t *testing.T) {
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -255,7 +258,8 @@ func TestInternalObjWrapFsFound(t *testing.T) {
 
 func TestInternalObjNotFound(t *testing.T) {
 	id := fmt.Sprintf("tionf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
@@ -265,7 +269,8 @@ func TestInternalObjNotFound(t *testing.T) {
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -292,7 +297,8 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	// write the object
 	runInstance.writeRemoteString(t, rootFs, "one", "one content")
@@ -310,7 +316,8 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	var err error
 
 	// create some rand test data
@@ -339,7 +346,8 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -369,7 +377,8 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -395,7 +404,8 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -449,7 +459,8 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 
 func TestInternalMoveWithNotify(t *testing.T) {
 	id := fmt.Sprintf("timwn%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -535,7 +546,8 @@ func TestInternalMoveWithNotify(t *testing.T) {
 
 func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	id := fmt.Sprintf("tincep%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -621,7 +633,8 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -653,7 +666,8 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"writes": "true"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -674,7 +688,8 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("timcsr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"workers": "1"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -709,7 +724,8 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
@@ -746,7 +762,9 @@ func TestInternalBug2117(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
+		map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	if runInstance.rootIsCrypt {
 		t.Skipf("skipping crypt")
@@ -847,7 +865,7 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 	return enc
 }
 
-func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
+func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
 	for _, s := range config.FileSections() {
@@ -940,15 +958,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
-
-	t.Cleanup(func() {
-		runInstance.cleanupFs(t, f)
-	})
-
 	return f, boltDb
 }
 
-func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
+func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
 	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
@@ -1098,6 +1111,27 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
+func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = in.Close()
+	}()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = out.Close()
+	}()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 

backend/cache/cache_test.go

@@ -18,7 +18,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
 		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})

backend/cache/cache_upload_test.go

@@ -21,8 +21,10 @@ import (
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	runInstance.newCacheFs(t, remoteName, id, false, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -61,7 +63,9 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -69,15 +73,19 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -111,8 +119,10 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -152,19 +162,21 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
 	minSize := 5242880
 	maxSize := 10485760
 	totalFiles := 10
-	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
+	rand.Seed(time.Now().Unix())
 
 	lastFile := ""
 	for i := 0; i < totalFiles; i++ {
-		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"
 		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
@@ -201,7 +213,9 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -329,7 +343,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 

backend/chunker/chunker.go

@@ -325,14 +325,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		}
 	}
 
-	// Correct root if definitely pointing to a file
-	if err == fs.ErrorIsFile {
-		f.root = path.Dir(f.root)
-		if f.root == "." || f.root == "/" {
-			f.root = ""
-		}
-	}
-
 	// Note 1: the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs.
 	// Note 2: features.Fill() points features.PutStream to our PutStream,

backend/chunker/chunker_test.go

@@ -40,7 +40,6 @@ func TestIntegration(t *testing.T) {
 		UnimplementableFsMethods: []string{
 			"PublicLink",
 			"OpenWriterAt",
-			"OpenChunkWriter",
 			"MergeDirs",
 			"DirCacheFlush",
 			"UserInfo",

backend/combine/combine.go

@@ -1,4 +1,4 @@
-// Package combine implements a backend to combine multiple remotes in a directory tree
+// Package combine implents a backend to combine multiple remotes in a directory tree
 package combine
 
 /*
@@ -233,7 +233,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            true,
-		PartialUploads:          true,
 	}).Fill(ctx, f)
 	canMove := true
 	for _, u := range f.upstreams {
@@ -290,16 +289,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 	}
 
-	// Enable CleanUp when any upstreams support it
-	if features.CleanUp == nil {
-		for _, u := range f.upstreams {
-			if u.f.Features().CleanUp != nil {
-				features.CleanUp = f.CleanUp
-				break
-			}
-		}
-	}
-
 	// Enable ChangeNotify when any upstreams support it
 	if features.ChangeNotify == nil {
 		for _, u := range f.upstreams {
@@ -310,9 +299,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 	}
 
-	// show that we wrap other backends
-	features.Overlay = true
-
 	f.features = features
 
 	// Get common intersection of hashes
@@ -365,7 +351,7 @@ func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream
 	return g.Wait()
 }
 
-// join the elements together but unlike path.Join return empty string
+// join the elements together but unline path.Join return empty string
 func join(elem ...string) string {
 	result := path.Join(elem...)
 	if result == "." {
@@ -645,7 +631,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 	if err != nil {
 		return nil, err
 	}
-	uSrc := fs.NewOverrideRemote(src, uRemote)
+	uSrc := operations.NewOverrideRemote(src, uRemote)
 	var o fs.Object
 	if stream {
 		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)
@@ -901,100 +887,6 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	})
 }
 
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	u, uRemote, err := f.findUpstream(remote)
-	if err != nil {
-		return "", err
-	}
-	do := u.f.Features().PublicLink
-	if do == nil {
-		return "", fs.ErrorNotImplemented
-	}
-	return do(ctx, uRemote, expire, unlink)
-}
-
-// PutUnchecked in to the remote path with the modTime given of the given size
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-//
-// May create duplicates or return errors if src already
-// exists.
-func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	srcPath := src.Remote()
-	u, uRemote, err := f.findUpstream(srcPath)
-	if err != nil {
-		return nil, err
-	}
-	do := u.f.Features().PutUnchecked
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	uSrc := fs.NewOverrideRemote(src, uRemote)
-	return do(ctx, in, uSrc, options...)
-}
-
-// MergeDirs merges the contents of all the directories passed
-// in into the first one and rmdirs the other directories.
-func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
-	if len(dirs) == 0 {
-		return nil
-	}
-	var (
-		u     *upstream
-		uDirs []fs.Directory
-	)
-	for _, dir := range dirs {
-		uNew, uDir, err := f.findUpstream(dir.Remote())
-		if err != nil {
-			return err
-		}
-		if u == nil {
-			u = uNew
-		} else if u != uNew {
-			return fmt.Errorf("can't merge directories from different upstreams")
-		}
-		uDirs = append(uDirs, fs.NewOverrideDirectory(dir, uDir))
-	}
-	do := u.f.Features().MergeDirs
-	if do == nil {
-		return fs.ErrorNotImplemented
-	}
-	return do(ctx, uDirs)
-}
-
-// CleanUp the trash in the Fs
-//
-// Implement this if you have a way of emptying the trash or
-// otherwise cleaning up old versions of files.
-func (f *Fs) CleanUp(ctx context.Context) error {
-	return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
-		if do := u.f.Features().CleanUp; do != nil {
-			return do(ctx)
-		}
-		return nil
-	})
-}
-
-// OpenWriterAt opens with a handle for random access writes
-//
-// Pass in the remote desired and the size if known.
-//
-// It truncates any existing object
-func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
-	u, uRemote, err := f.findUpstream(remote)
-	if err != nil {
-		return nil, err
-	}
-	do := u.f.Features().OpenWriterAt
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	return do(ctx, uRemote, size)
-}
-
 // Object describes a wrapped Object
 //
 // This is a wrapped Object which knows its path prefix
@@ -1024,7 +916,7 @@ func (o *Object) String() string {
 func (o *Object) Remote() string {
 	newPath, err := o.u.pathAdjustment.do(o.Object.String())
 	if err != nil {
-		fs.Errorf(o.Object, "Bad object: %v", err)
+		fs.Errorf(o, "Bad object: %v", err)
 		return err.Error()
 	}
 	return newPath
@@ -1096,10 +988,5 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
-	_ fs.PutUncheckeder  = (*Fs)(nil)
-	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.CleanUpper      = (*Fs)(nil)
-	_ fs.OpenWriterAter  = (*Fs)(nil)
 	_ fs.FullObject      = (*Object)(nil)
 )

backend/combine/combine_test.go

@@ -10,11 +10,6 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
-var (
-	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "OpenChunkWriter"}
-	unimplementableObjectMethods = []string{}
-)
-
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	if *fstest.RemoteName == "" {
@@ -22,8 +17,8 @@ func TestIntegration(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -41,8 +36,6 @@ func TestLocal(t *testing.T) {
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
 		QuickTestOK: true,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -59,8 +52,6 @@ func TestMemory(t *testing.T) {
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
 		QuickTestOK: true,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -77,8 +68,6 @@ func TestMixed(t *testing.T) {
 			{Name: name, Key: "type", Value: "combine"},
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 

backend/compress/compress.go

@@ -14,7 +14,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
 	"regexp"
 	"strings"
 	"time"
@@ -173,13 +172,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		opt:  *opt,
 		mode: compressionModeFromName(opt.CompressionMode),
 	}
-	// Correct root if definitely pointing to a file
-	if err == fs.ErrorIsFile {
-		f.root = path.Dir(f.root)
-		if f.root == "." || f.root == "/" {
-			f.root = ""
-		}
-	}
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
@@ -194,7 +186,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            true,
-		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -265,16 +256,6 @@ func isMetadataFile(filename string) bool {
 	return strings.HasSuffix(filename, metaFileExt)
 }
 
-// Checks whether a file is a metadata file and returns the original
-// file name and a flag indicating whether it was a metadata file or
-// not.
-func unwrapMetadataFile(filename string) (string, bool) {
-	if !isMetadataFile(filename) {
-		return "", false
-	}
-	return filename[:len(filename)-len(metaFileExt)], true
-}
-
 // makeDataName generates the file name for a data file with specified compression mode
 func makeDataName(remote string, size int64, mode int) (newRemote string) {
 	if mode != Uncompressed {
@@ -998,7 +979,6 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		fs.Logf(f, "path %q entryType %d", path, entryType)
 		var (
 			wrappedPath string
-			isMetadataFile bool
 		)
 		switch entryType {
 		case fs.EntryDirectory:
@@ -1006,10 +986,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		case fs.EntryObject:
 			// Note: All we really need to do to monitor the object is to check whether the metadata changed,
 			// as the metadata contains the hash. This will work unless there's a hash collision and the sizes stay the same.
-			wrappedPath, isMetadataFile = unwrapMetadataFile(path)
-			if !isMetadataFile {
-				return
-			}
+			wrappedPath = makeMetadataName(path)
 		default:
 			fs.Errorf(path, "press ChangeNotify: ignoring unknown EntryType %d", entryType)
 			return

backend/compress/compress_test.go

@@ -14,12 +14,13 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
-var defaultOpt = fstests.Opt{
-	RemoteName: "TestCompress:",
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	opt := fstests.Opt{
+		RemoteName: *fstest.RemoteName,
 		NilObject:  (*Object)(nil),
 		UnimplementableFsMethods: []string{
 			"OpenWriterAt",
-		"OpenChunkWriter",
 			"MergeDirs",
 			"DirCacheFlush",
 			"PutUnchecked",
@@ -28,12 +29,8 @@ var defaultOpt = fstests.Opt{
 			"Disconnect",
 		},
 		TiersToTest:                  []string{"STANDARD", "STANDARD_IA"},
-	UnimplementableObjectMethods: []string{},
-}
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &defaultOpt)
+		UnimplementableObjectMethods: []string{}}
+	fstests.Run(t, &opt)
 }
 
 // TestRemoteGzip tests GZIP compression
@@ -43,13 +40,27 @@ func TestRemoteGzip(t *testing.T) {
 	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-gzip")
 	name := "TestCompressGzip"
-	opt := defaultOpt
-	opt.RemoteName = name + ":"
-	opt.ExtraConfig = []fstests.ExtraConfigItem{
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		UnimplementableFsMethods: []string{
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"PutUnchecked",
+			"PutStream",
+			"UserInfo",
+			"Disconnect",
+		},
+		UnimplementableObjectMethods: []string{
+			"GetTier",
+			"SetTier",
+		},
+		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "type", Value: "compress"},
 			{Name: name, Key: "remote", Value: tempdir},
 			{Name: name, Key: "compression_mode", Value: "gzip"},
-	}
-	opt.QuickTestOK = true
-	fstests.Run(t, &opt)
+		},
+		QuickTestOK: true,
+	})
 }

backend/crypt/cipher.go

@@ -21,7 +21,6 @@ import (
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
@@ -38,6 +37,7 @@ const (
 	blockHeaderSize     = secretbox.Overhead
 	blockDataSize       = 64 * 1024
 	blockSize           = blockHeaderSize + blockDataSize
+	encryptedSuffix     = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
 )
 
 // Errors returned by cipher
@@ -53,9 +53,8 @@ var (
 	ErrorEncryptedBadBlock       = errors.New("failed to authenticate decrypted block - bad password?")
 	ErrorBadBase32Encoding       = errors.New("bad base32 filename encoding")
 	ErrorFileClosed              = errors.New("file already closed")
-	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - does not match suffix")
+	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
 	ErrorBadSeek                 = errors.New("Seek beyond end of file")
-	ErrorSuffixMissingDot        = errors.New("suffix config setting should include a '.'")
 	defaultSalt                  = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
 	obfuscQuoteRune              = '!'
 )
@@ -179,8 +178,6 @@ type Cipher struct {
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
-	passBadBlocks   bool // if set passed bad blocks as zeroed blocks
-	encryptedSuffix string
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -190,10 +187,9 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 		fileNameEnc:    enc,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
-		encryptedSuffix: ".bin",
 	}
 	c.buffers.New = func() interface{} {
-		return new([blockSize]byte)
+		return make([]byte, blockSize)
 	}
 	err := c.Key(password, salt)
 	if err != nil {
@@ -202,29 +198,11 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
-// setEncryptedSuffix set suffix, or an empty string
-func (c *Cipher) setEncryptedSuffix(suffix string) {
-	if strings.EqualFold(suffix, "none") {
-		c.encryptedSuffix = ""
-		return
-	}
-	if !strings.HasPrefix(suffix, ".") {
-		fs.Errorf(nil, "crypt: bad suffix: %v", ErrorSuffixMissingDot)
-		suffix = "." + suffix
-	}
-	c.encryptedSuffix = suffix
-}
-
-// Call to set bad block pass through
-func (c *Cipher) setPassBadBlocks(passBadBlocks bool) {
-	c.passBadBlocks = passBadBlocks
-}
-
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
 // If salt is "" we use a fixed salt just to make attackers lives
-// slightly harder than using no salt.
+// slighty harder than using no salt.
 //
 // Note that empty password makes all 0x00 keys which is used in the
 // tests.
@@ -252,12 +230,15 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() *[blockSize]byte {
-	return c.buffers.Get().(*[blockSize]byte)
+func (c *Cipher) getBlock() []byte {
+	return c.buffers.Get().([]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf *[blockSize]byte) {
+func (c *Cipher) putBlock(buf []byte) {
+	if len(buf) != blockSize {
+		panic("bad blocksize returned to pool")
+	}
 	c.buffers.Put(buf)
 }
 
@@ -527,7 +508,7 @@ func (c *Cipher) encryptFileName(in string) string {
 // EncryptFileName encrypts a file path
 func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
-		return in + c.encryptedSuffix
+		return in + encryptedSuffix
 	}
 	return c.encryptFileName(in)
 }
@@ -587,8 +568,8 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 // DecryptFileName decrypts a file path
 func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
-		remainingLength := len(in) - len(c.encryptedSuffix)
-		if remainingLength == 0 || !strings.HasSuffix(in, c.encryptedSuffix) {
+		remainingLength := len(in) - len(encryptedSuffix)
+		if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
 			return "", ErrorNotAnEncryptedFile
 		}
 		decrypted := in[:remainingLength]
@@ -628,7 +609,7 @@ func (n *nonce) pointer() *[fileNonceSize]byte {
 // fromReader fills the nonce from an io.Reader - normally the OSes
 // crypto random number generator
 func (n *nonce) fromReader(in io.Reader) error {
-	read, err := readers.ReadFill(in, (*n)[:])
+	read, err := io.ReadFull(in, (*n)[:])
 	if read != fileNonceSize {
 		return fmt.Errorf("short read of nonce: %w", err)
 	}
@@ -683,8 +664,8 @@ type encrypter struct {
 	in       io.Reader
 	c        *Cipher
 	nonce    nonce
-	buf      *[blockSize]byte
-	readBuf  *[blockSize]byte
+	buf      []byte
+	readBuf  []byte
 	bufIndex int
 	bufSize  int
 	err      error
@@ -709,9 +690,9 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 		}
 	}
 	// Copy magic into buffer
-	copy((*fh.buf)[:], fileMagicBytes)
+	copy(fh.buf, fileMagicBytes)
 	// Copy nonce into buffer
-	copy((*fh.buf)[fileMagicSize:], fh.nonce[:])
+	copy(fh.buf[fileMagicSize:], fh.nonce[:])
 	return fh, nil
 }
 
@@ -726,20 +707,22 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 	if fh.bufIndex >= fh.bufSize {
 		// Read data
 		// FIXME should overlap the reads with a go-routine and 2 buffers?
-		readBuf := (*fh.readBuf)[:blockDataSize]
-		n, err = readers.ReadFill(fh.in, readBuf)
+		readBuf := fh.readBuf[:blockDataSize]
+		n, err = io.ReadFull(fh.in, readBuf)
 		if n == 0 {
+			// err can't be nil since:
+			// n == len(buf) if and only if err == nil.
 			return fh.finish(err)
 		}
 		// possibly err != nil here, but we will process the
-		// data and the next call to ReadFill will return 0, err
+		// data and the next call to ReadFull will return 0, err
 		// Encrypt the block using the nonce
-		secretbox.Seal((*fh.buf)[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
 	}
-	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufSize])
+	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
 	fh.bufIndex += n
 	return n, nil
 }
@@ -780,8 +763,8 @@ type decrypter struct {
 	nonce        nonce
 	initialNonce nonce
 	c            *Cipher
-	buf          *[blockSize]byte
-	readBuf      *[blockSize]byte
+	buf          []byte
+	readBuf      []byte
 	bufIndex     int
 	bufSize      int
 	err          error
@@ -799,12 +782,12 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 		limit:   -1,
 	}
 	// Read file header (magic + nonce)
-	readBuf := (*fh.readBuf)[:fileHeaderSize]
-	n, err := readers.ReadFill(fh.rc, readBuf)
-	if n < fileHeaderSize && err == io.EOF {
+	readBuf := fh.readBuf[:fileHeaderSize]
+	_, err := io.ReadFull(fh.rc, readBuf)
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
 		// This read from 0..fileHeaderSize-1 bytes
 		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
-	} else if err != io.EOF && err != nil {
+	} else if err != nil {
 		return nil, fh.finishAndClose(err)
 	}
 	// check the magic
@@ -862,8 +845,10 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 func (fh *decrypter) fillBuffer() (err error) {
 	// FIXME should overlap the reads with a go-routine and 2 buffers?
 	readBuf := fh.readBuf
-	n, err := readers.ReadFill(fh.rc, (*readBuf)[:])
+	n, err := io.ReadFull(fh.rc, readBuf)
 	if n == 0 {
+		// err can't be nil since:
+		// n == len(buf) if and only if err == nil.
 		return err
 	}
 	// possibly err != nil here, but we will process the data and
@@ -871,26 +856,19 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	// Check header + 1 byte exists
 	if n <= blockHeaderSize {
-		if err != nil && err != io.EOF {
+		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open((*fh.buf)[:0], (*readBuf)[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
-		if err != nil && err != io.EOF {
+		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
-		if !fh.c.passBadBlocks {
 		return ErrorEncryptedBadBlock
 	}
-		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
-		// Zero out the bad block and continue
-		for i := range (*fh.buf)[:n] {
-			(*fh.buf)[i] = 0
-		}
-	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize
 	fh.nonce.increment()
@@ -915,7 +893,7 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	if fh.limit >= 0 && fh.limit < int64(toCopy) {
 		toCopy = int(fh.limit)
 	}
-	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufIndex+toCopy])
+	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
 	fh.bufIndex += n
 	if fh.limit >= 0 {
 		fh.limit -= int64(n)
@@ -926,8 +904,9 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-// calculateUnderlying converts an (offset, limit) in an encrypted file
-// into an (underlyingOffset, underlyingLimit) for the underlying file.
+// calculateUnderlying converts an (offset, limit) in a crypted file
+// into an (underlyingOffset, underlyingLimit) for the underlying
+// file.
 //
 // It also returns number of bytes to discard after reading the first
 // block and number of blocks this is from the start so the nonce can

backend/crypt/cipher_test.go

@@ -27,14 +27,14 @@ func TestNewNameEncryptionMode(t *testing.T) {
 		{"off", NameEncryptionOff, ""},
 		{"standard", NameEncryptionStandard, ""},
 		{"obfuscate", NameEncryptionObfuscated, ""},
-		{"potato", NameEncryptionOff, "unknown file name encryption mode \"potato\""},
+		{"potato", NameEncryptionOff, "Unknown file name encryption mode \"potato\""},
 	} {
 		actual, actualErr := NewNameEncryptionMode(test.in)
 		assert.Equal(t, actual, test.expected)
 		if test.expectedErr == "" {
 			assert.NoError(t, actualErr)
 		} else {
-			assert.EqualError(t, actualErr, test.expectedErr)
+			assert.Error(t, actualErr, test.expectedErr)
 		}
 	}
 }
@@ -405,13 +405,6 @@ func TestNonStandardEncryptFileName(t *testing.T) {
 	// Off mode
 	c, _ := newCipher(NameEncryptionOff, "", "", true, nil)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
-	// Off mode with custom suffix
-	c, _ = newCipher(NameEncryptionOff, "", "", true, nil)
-	c.setEncryptedSuffix(".jpg")
-	assert.Equal(t, "1/12/123.jpg", c.EncryptFileName("1/12/123"))
-	// Off mode with empty suffix
-	c.setEncryptedSuffix("none")
-	assert.Equal(t, "1/12/123", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", true, nil)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
@@ -490,27 +483,21 @@ func TestNonStandardDecryptFileName(t *testing.T) {
 			in             string
 			expected       string
 			expectedErr    error
-			customSuffix   string
 		}{
-			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil, ""},
-			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile, ""},
-			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile, ""},
-			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil, ""},
-			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil, ""},
-			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil, ""},
-			{NameEncryptionOff, true, "1/12/123.jpg", "1/12/123", nil, ".jpg"},
-			{NameEncryptionOff, true, "1/12/123", "1/12/123", nil, "none"},
-			{NameEncryptionObfuscated, true, "!.hello", "hello", nil, ""},
-			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile, ""},
-			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil, ""},
-			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil, ""},
-			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil, ""},
-			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil, ""},
+			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
+			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
+			{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
+			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
+			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
 		} {
 			c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
-			if test.customSuffix != "" {
-				c.setEncryptedSuffix(test.customSuffix)
-			}
 			actual, actualErr := c.DecryptFileName(test.in)
 			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
 			assert.Equal(t, test.expected, actual, what)
@@ -739,7 +726,7 @@ func TestNonceFromReader(t *testing.T) {
 	assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
 	buf = bytes.NewBufferString("123456789abcdefghijklmn")
 	err = x.fromReader(buf)
-	assert.EqualError(t, err, "short read of nonce: EOF")
+	assert.Error(t, err, "short read of nonce")
 }
 
 func TestNonceFromBuf(t *testing.T) {
@@ -1063,7 +1050,7 @@ func TestRandomSource(t *testing.T) {
 	_, _ = source.Read(buf)
 	sink = newRandomSource(1e8)
 	_, err = io.Copy(sink, source)
-	assert.EqualError(t, err, "Error in stream at 1")
+	assert.Error(t, err, "Error in stream")
 }
 
 type zeroes struct{}
@@ -1180,13 +1167,13 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err := c.newEncrypter(z, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
-	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, (*fh.buf)[:32])
+	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
 
 	// Test error path
 	c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
-	assert.EqualError(t, err, "short read of nonce: EOF")
+	assert.Error(t, err, "short read of nonce")
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
@@ -1237,7 +1224,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
 		fh, err = c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
+		assert.Error(t, err, ErrorEncryptedFileTooShort.Error())
 		assert.Equal(t, 1, cd.closed)
 	}
 
@@ -1245,7 +1232,7 @@ func TestNewDecrypter(t *testing.T) {
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
-	assert.EqualError(t, err, "potato")
+	assert.Error(t, err, "potato")
 	assert.Equal(t, 1, cd.closed)
 
 	// bad magic
@@ -1256,7 +1243,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0copy))
 		fh, err := c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
+		assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 		file0copy[i] ^= 0x1
 		assert.Equal(t, 1, cd.closed)
 	}
@@ -1508,10 +1495,8 @@ func TestDecrypterRead(t *testing.T) {
 		case i == fileHeaderSize:
 			// This would normally produce an error *except* on the first block
 			expectedErr = nil
-		case i <= fileHeaderSize+blockHeaderSize:
-			expectedErr = ErrorEncryptedFileBadHeader
 		default:
-			expectedErr = ErrorEncryptedBadBlock
+			expectedErr = io.ErrUnexpectedEOF
 		}
 		if expectedErr != nil {
 			assert.EqualError(t, err, expectedErr.Error(), what)
@@ -1529,7 +1514,7 @@ func TestDecrypterRead(t *testing.T) {
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
 	_, err = io.ReadAll(fh)
-	assert.EqualError(t, err, "potato")
+	assert.Error(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
 	// Test corrupting the input
@@ -1540,26 +1525,15 @@ func TestDecrypterRead(t *testing.T) {
 		file16copy[i] ^= 0xFF
 		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
-			assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
+			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
 			_, err = io.ReadAll(fh)
-			assert.EqualError(t, err, ErrorEncryptedBadBlock.Error())
+			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
 		}
 		file16copy[i] ^= 0xFF
 	}
-
-	// Test that we can corrupt a byte and read zeroes if
-	// passBadBlocks is set
-	copy(file16copy, file16)
-	file16copy[len(file16copy)-1] ^= 0xFF
-	c.passBadBlocks = true
-	fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
-	assert.NoError(t, err)
-	buf, err := io.ReadAll(fh)
-	assert.NoError(t, err)
-	assert.Equal(t, make([]byte, 16), buf)
 }
 
 func TestDecrypterClose(t *testing.T) {
@@ -1580,7 +1554,7 @@ func TestDecrypterClose(t *testing.T) {
 
 	// double close
 	err = fh.Close()
-	assert.EqualError(t, err, ErrorFileClosed.Error())
+	assert.Error(t, err, ErrorFileClosed.Error())
 	assert.Equal(t, 1, cd.closed)
 
 	// try again reading the file this time
@@ -1607,6 +1581,8 @@ func TestPutGetBlock(t *testing.T) {
 	block := c.getBlock()
 	c.putBlock(block)
 	c.putBlock(block)
+
+	assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) })
 }
 
 func TestKey(t *testing.T) {

backend/crypt/crypt.go

@@ -48,7 +48,7 @@ func init() {
 					Help:  "Very simple filename obfuscation.",
 				}, {
 					Value: "off",
-					Help:  "Don't encrypt the file names.\nAdds a \".bin\", or \"suffix\" extension only.",
+					Help:  "Don't encrypt the file names.\nAdds a \".bin\" extension only.",
 				},
 			},
 		}, {
@@ -79,9 +79,7 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 		}, {
 			Name:    "server_side_across_configs",
 			Default: false,
-			Help: `Deprecated: use --server-side-across-configs instead.
-
-Allow server-side operations (e.g. copy) to work across different crypt configs.
+			Help: `Allow server-side operations (e.g. copy) to work across different crypt configs.
 
 Normally this option is not what you want, but if you have two crypts
 pointing to the same backend you can use it.
@@ -121,15 +119,6 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
-		}, {
-			Name: "pass_bad_blocks",
-			Help: `If set this will pass bad blocks through as all 0.
-
-This should not be set in normal operation, it should only be set if
-trying to recover an encrypted file with errors and it is desired to
-recover as much of the file as possible.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -149,18 +138,10 @@ length and if it's case sensitive.`,
 				},
 				{
 					Value: "base32768",
-					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox)",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
 				},
 			},
 			Advanced: true,
-		}, {
-			Name: "suffix",
-			Help: `If this is set it will override the default suffix of ".bin".
-
-Setting suffix to "none" will result in an empty suffix. This may be useful 
-when the path length is critical.`,
-			Default:  ".bin",
-			Advanced: true,
 		}},
 	})
 }
@@ -193,8 +174,6 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
-	cipher.setEncryptedSuffix(opt.Suffix)
-	cipher.setPassBadBlocks(opt.PassBadBlocks)
 	return cipher, nil
 }
 
@@ -253,17 +232,10 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		cipher: cipher,
 	}
 	cache.PinUntilFinalized(f.Fs, f)
-	// Correct root if definitely pointing to a file
-	if err == fs.ErrorIsFile {
-		f.root = path.Dir(f.root)
-		if f.root == "." || f.root == "/" {
-			f.root = ""
-		}
-	}
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
+		CaseInsensitive:         cipher.NameEncryptionMode() == NameEncryptionOff,
 		DuplicateFiles:          true,
 		ReadMimeType:            false, // MimeTypes not supported with crypt
 		WriteMimeType:           false,
@@ -275,7 +247,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            true,
-		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -291,9 +262,7 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
-	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
-	Suffix                  string `config:"suffix"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -427,8 +396,6 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
-	ci := fs.GetConfig(ctx)
-
 	if f.opt.NoDataEncryption {
 		o, err := put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
 		if err == nil && o != nil {
@@ -446,9 +413,6 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	// Find a hash the destination supports to compute a hash of
 	// the encrypted data
 	ht := f.Fs.Hashes().GetOne()
-	if ci.IgnoreChecksum {
-		ht = hash.None
-	}
 	var hasher *hash.MultiHasher
 	if ht != hash.None {
 		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
@@ -485,7 +449,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v crypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -1083,11 +1047,10 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	// Get the underlying object if there is one
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(*fs.OverrideRemote); ok {
-		// Unwrap if it is an operations.OverrideRemote
+	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
+		// Otherwise likely is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
-		// Otherwise don't unwrap any further
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash

backend/crypt/crypt_internal_test.go

@@ -17,28 +17,41 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type testWrapper struct {
+	fs.ObjectInfo
+}
+
+// UnWrap returns the Object that this Object is wrapping or nil if it
+// isn't wrapping anything
+func (o testWrapper) UnWrap() fs.Object {
+	if o, ok := o.ObjectInfo.(fs.Object); ok {
+		return o
+	}
+	return nil
+}
+
 // Create a temporary local fs to upload things from
 
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
 	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
-	t.Cleanup(func() {
+	cleanup = func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	})
-	return localFs
+	}
+	return localFs, cleanup
 }
 
 // Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
 	inBuf := bytes.NewBufferString(contents)
 	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
 	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
 	obj, err := f.Put(context.Background(), inBuf, upSrc)
 	require.NoError(t, err)
-	t.Cleanup(func() {
+	cleanup = func() {
 		require.NoError(t, obj.Remove(context.Background()))
-	})
-	return obj
+	}
+	return obj, cleanup
 }
 
 // Test the ObjectInfo
@@ -52,9 +65,11 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 		path = "_wrap"
 	}
 
-	localFs := makeTempLocalFs(t)
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
 
-	obj := uploadFile(t, localFs, path, contents)
+	obj, cleanupObj := uploadFile(t, localFs, path, contents)
+	defer cleanupObj()
 
 	// encrypt the data
 	inBuf := bytes.NewBufferString(contents)
@@ -68,7 +83,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	var oi fs.ObjectInfo = obj
 	if wrap {
 		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = fs.NewOverrideRemote(oi, "new_remote")
+		oi = testWrapper{oi}
 	}
 
 	// wrap the object in a crypt for upload using the nonce we
@@ -101,13 +116,16 @@ func testComputeHash(t *testing.T, f *Fs) {
 		t.Skipf("%v: does not support hashes", f.Fs)
 	}
 
-	localFs := makeTempLocalFs(t)
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
 
 	// Upload a file to localFs as a test object
-	localObj := uploadFile(t, localFs, path, contents)
+	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
+	defer cleanupLocalObj()
 
 	// Upload the same data to the remote Fs also
-	remoteObj := uploadFile(t, f, path, contents)
+	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
+	defer cleanupRemoteObj()
 
 	// Calculate the expected Hash of the remote object
 	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)

backend/crypt/crypt_test.go

@@ -24,7 +24,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
 		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -45,7 +45,7 @@ func TestStandardBase32(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -67,7 +67,7 @@ func TestStandardBase64(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base64"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -89,7 +89,7 @@ func TestStandardBase32768(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base32768"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -111,7 +111,7 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -137,7 +137,7 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -164,7 +164,7 @@ func TestNoDataObfuscate(t *testing.T) {
 			{Name: name, Key: "no_data_encryption", Value: "true"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})

backend/drive/drive.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -71,7 +72,7 @@ const (
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	minChunkSize     = fs.SizeSuffix(googleapi.MinUploadChunkSize)
 	defaultChunkSize = 8 * fs.Mebi
-	partialFields    = "id,name,size,md5Checksum,sha1Checksum,sha256Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
+	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
 	defaultXDGIcon   = "text-html"
@@ -143,41 +144,6 @@ var (
 	_linkTemplates   map[string]*template.Template // available link types
 )
 
-// rwChoices type for fs.Bits
-type rwChoices struct{}
-
-func (rwChoices) Choices() []fs.BitsChoicesInfo {
-	return []fs.BitsChoicesInfo{
-		{Bit: uint64(rwOff), Name: "off"},
-		{Bit: uint64(rwRead), Name: "read"},
-		{Bit: uint64(rwWrite), Name: "write"},
-	}
-}
-
-// rwChoice type alias
-type rwChoice = fs.Bits[rwChoices]
-
-const (
-	rwRead rwChoice = 1 << iota
-	rwWrite
-	rwOff rwChoice = 0
-)
-
-// Examples for the options
-var rwExamples = fs.OptionExamples{{
-	Value: rwOff.String(),
-	Help:  "Do not read or write the value",
-}, {
-	Value: rwRead.String(),
-	Help:  "Read the value only",
-}, {
-	Value: rwWrite.String(),
-	Help:  "Write the value only",
-}, {
-	Value: (rwRead | rwWrite).String(),
-	Help:  "Read and Write the value.",
-}}
-
 // Parse the scopes option returning a slice of scopes
 func driveScopes(scopesString string) (scopes []string) {
 	if scopesString == "" {
@@ -237,7 +203,7 @@ func init() {
 					m.Set("root_folder_id", "appDataFolder")
 				}
 
-				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
+				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
 					return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
 						OAuth2Config: driveConfig,
 					})
@@ -285,13 +251,9 @@ func init() {
 			}
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
-		MetadataInfo: &fs.MetadataInfo{
-			System: systemMetadataInfo,
-			Help:   `User metadata is stored in the properties field of the drive object.`,
-		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
-			Help: "Comma separated list of scopes that rclone should use when requesting access from drive.",
+			Help: "Scope that rclone should use when requesting access from drive.",
 			Examples: []fs.OptionExample{{
 				Value: "drive",
 				Help:  "Full access all files, excluding Application Data Folder.",
@@ -317,7 +279,6 @@ Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 `,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
@@ -326,13 +287,11 @@ a non root folder as its starting point.
 			Help:     "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name:     "team_drive",
 			Help:     "ID of the Shared Drive (Team Drive).",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name:     "auth_owner_only",
 			Default:  false,
@@ -359,35 +318,16 @@ rather than shortcuts themselves when doing server side copies.`,
 			Default:  false,
 			Help:     "Skip google documents in all listings.\n\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
-		}, {
-			Name:    "show_all_gdocs",
-			Default: false,
-			Help: `Show all Google Docs including non-exportable ones in listings.
-
-If you try a server side copy on a Google Form without this flag, you
-will get this error:
-
-    No export formats found for "application/vnd.google-apps.form"
-
-However adding this flag will allow the form to be server side copied.
-
-Note that rclone doesn't add extensions to the Google Docs file names
-in this mode.
-
-Do **not** use this flag when trying to download Google Docs - rclone
-will fail to download them.
-`,
-			Advanced: true,
 		}, {
 			Name:    "skip_checksum_gphotos",
 			Default: false,
-			Help: `Skip checksums on Google photos and videos only.
+			Help: `Skip MD5 checksum on Google photos and videos only.
 
 Use this if you get checksum errors when transferring Google photos or
 videos.
 
 Setting this flag will cause Google photos and videos to return a
-blank checksums.
+blank MD5 checksum.
 
 Google photos are identified by being in the "photos" space.
 
@@ -481,7 +421,6 @@ date is used.`,
 			Default:  "",
 			Help:     `Impersonate this user when using a service account.`,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name:    "alternate_export",
 			Default: false,
@@ -513,11 +452,7 @@ If downloading a file returns the error "This file has been identified
 as malware or spam and cannot be downloaded" with the error code
 "cannotDownloadAbusiveFile" then supply this flag to rclone to
 indicate you acknowledge the risks of downloading the file and rclone
-will download it anyway.
-
-Note that if you are using service account it will need Manager
-permission (not Content Manager) to for this flag to work. If the SA
-does not have the right permission, Google will just ignore the flag.`,
+will download it anyway.`,
 			Advanced: true,
 		}, {
 			Name:     "keep_revision_forever",
@@ -561,9 +496,7 @@ need to use --ignore size also.`,
 		}, {
 			Name:    "server_side_across_configs",
 			Default: false,
-			Help: `Deprecated: use --server-side-across-configs instead.
-
-Allow server-side operations (e.g. copy) to work across different drive configs.
+			Help: `Allow server-side operations (e.g. copy) to work across different drive configs.
 
 This can be useful if you wish to do a server-side copy between two
 different Google drives.  Note that this isn't enabled by default
@@ -652,82 +585,9 @@ This resource key requirement only applies to a subset of old files.
 
 Note also that opening the folder once in the web interface (with the
 user you've authenticated rclone with) seems to be enough so that the
-resource key is not needed.
+resource key is no needed.
 `,
 			Advanced: true,
-			Sensitive: true,
-		}, {
-			Name: "fast_list_bug_fix",
-			Help: `Work around a bug in Google Drive listing.
-
-Normally rclone will work around a bug in Google Drive when using
---fast-list (ListR) where the search "(A in parents) or (B in
-parents)" returns nothing sometimes. See #3114, #4289 and
-https://issuetracker.google.com/issues/149522397
-
-Rclone detects this by finding no items in more than one directory
-when listing and retries them as lists of individual directories.
-
-This means that if you have a lot of empty directories rclone will end
-up listing them all individually and this can take many more API
-calls.
-
-This flag allows the work-around to be disabled. This is **not**
-recommended in normal use - only if you have a particular case you are
-having trouble with like many empty directories.
-`,
-			Advanced: true,
-			Default:  true,
-		}, {
-			Name: "metadata_owner",
-			Help: `Control whether owner should be read or written in metadata.
-
-Owner is a standard part of the file metadata so is easy to read. But it
-isn't always desirable to set the owner from the metadata.
-
-Note that you can't set the owner on Shared Drives, and that setting
-ownership will generate an email to the new owner (this can't be
-disabled), and you can't transfer ownership to someone outside your
-organization.
-`,
-			Advanced: true,
-			Default:  rwRead,
-			Examples: rwExamples,
-		}, {
-			Name: "metadata_permissions",
-			Help: `Control whether permissions should be read or written in metadata.
-
-Reading permissions metadata from files can be done quickly, but it
-isn't always desirable to set the permissions from the metadata.
-
-Note that rclone drops any inherited permissions on Shared Drives and
-any owner permission on My Drives as these are duplicated in the owner
-metadata.
-`,
-			Advanced: true,
-			Default:  rwOff,
-			Examples: rwExamples,
-		}, {
-			Name: "metadata_labels",
-			Help: `Control whether labels should be read or written in metadata.
-
-Reading labels metadata from files takes an extra API transaction and
-will slow down listings. It isn't always desirable to set the labels
-from the metadata.
-
-The format of labels is documented in the drive API documentation at
-https://developers.google.com/drive/api/reference/rest/v3/Label -
-rclone just provides a JSON dump of this format.
-
-When setting labels, the label and fields must already exist - rclone
-will not create them. This means that if you are transferring labels
-from two different accounts you will have to create the labels in
-advance and use the metadata mapper to translate the IDs between the
-two accounts.
-`,
-			Advanced: true,
-			Default:  rwOff,
-			Examples: rwExamples,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -735,18 +595,6 @@ two accounts.
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// Don't encode / as it's a valid name character in drive.
 			Default: encoder.EncodeInvalidUtf8,
-		}, {
-			Name:     "env_auth",
-			Help:     "Get IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
-			Default:  false,
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: "false",
-				Help:  "Enter credentials in the next step.",
-			}, {
-				Value: "true",
-				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
-			}},
 		}}...),
 	})
 
@@ -775,7 +623,6 @@ type Options struct {
 	UseTrash                  bool                 `config:"use_trash"`
 	CopyShortcutContent       bool                 `config:"copy_shortcut_content"`
 	SkipGdocs                 bool                 `config:"skip_gdocs"`
-	ShowAllGdocs              bool                 `config:"show_all_gdocs"`
 	SkipChecksumGphotos       bool                 `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool                 `config:"shared_with_me"`
 	TrashedOnly               bool                 `config:"trashed_only"`
@@ -803,12 +650,7 @@ type Options struct {
 	SkipShortcuts             bool                 `config:"skip_shortcuts"`
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
 	ResourceKey               string               `config:"resource_key"`
-	FastListBugFix            bool                 `config:"fast_list_bug_fix"`
-	MetadataOwner             rwChoice             `config:"metadata_owner"`
-	MetadataPermissions       rwChoice             `config:"metadata_permissions"`
-	MetadataLabels            rwChoice             `config:"metadata_labels"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote drive server
@@ -828,13 +670,12 @@ type Fs struct {
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
+	fileFields       googleapi.Field    // fields to fetch file info with
 	m                configmap.Mapper
 	grouping         int32               // number of IDs to search at once in ListR - read with atomic
 	listRmu          *sync.Mutex         // protects listRempties
 	listRempties     map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
 	dirResourceKeys  *sync.Map           // map directory ID to resource key
-	permissionsMu    *sync.Mutex                  // protect the below
-	permissions      map[string]*drive.Permission // map permission IDs to Permissions
 }
 
 type baseObject struct {
@@ -846,7 +687,6 @@ type baseObject struct {
 	bytes        int64    // size of the object
 	parents      []string // IDs of the parent directories
 	resourceKey  *string  // resourceKey is needed for link shared objects
-	metadata     *fs.Metadata // metadata if known
 }
 type documentObject struct {
 	baseObject
@@ -865,8 +705,6 @@ type Object struct {
 	baseObject
 	url        string // Download URL of this object
 	md5sum     string // md5sum of the object
-	sha1sum    string // sha1sum of the object
-	sha256sum  string // sha256sum of the object
 	v2Download bool   // generate v2 download link ondemand
 }
 
@@ -920,7 +758,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
-			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
+			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
 				fs.Errorf(f, "Received upload limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
@@ -1095,7 +933,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader)
 	}
 
-	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
+	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields)
 
 OUTER:
 	for {
@@ -1281,12 +1119,6 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 		if err != nil {
 			return nil, fmt.Errorf("failed to create oauth client from service account: %w", err)
 		}
-	} else if opt.EnvAuth {
-		scopes := driveScopes(opt.Scope)
-		oAuthClient, err = google.DefaultClient(ctx, scopes...)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create client from environment: %w", err)
-		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
 		if err != nil {
@@ -1369,10 +1201,9 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		listRmu:         new(sync.Mutex),
 		listRempties:    make(map[string]struct{}),
 		dirResourceKeys: new(sync.Map),
-		permissionsMu:   new(sync.Mutex),
-		permissions:     make(map[string]*drive.Permission),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
+	f.fileFields = f.getFileFields()
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
 		ReadMimeType:            true,
@@ -1380,9 +1211,6 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 		FilterAware:             true,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
@@ -1487,7 +1315,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
-func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File) (o baseObject, err error) {
+func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 	modifiedDate := info.ModifiedTime
 	if f.opt.UseCreatedDate {
 		modifiedDate = info.CreatedTime
@@ -1498,7 +1326,7 @@ func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File)
 	if f.opt.SizeAsQuota {
 		size = info.QuotaBytesUsed
 	}
-	o = baseObject{
+	return baseObject{
 		fs:           f,
 		remote:       remote,
 		id:           info.Id,
@@ -1507,15 +1335,10 @@ func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File)
 		bytes:        size,
 		parents:      info.Parents,
 	}
-	err = nil
-	if fs.GetConfig(ctx).Metadata {
-		err = o.parseMetadata(ctx, info)
-	}
-	return o, err
 }
 
 // getFileFields gets the fields for a normal file Get or List
-func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
+func (f *Fs) getFileFields() (fields googleapi.Field) {
 	fields = partialFields
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
@@ -1529,53 +1352,40 @@ func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
 	if f.opt.SizeAsQuota {
 		fields += ",quotaBytesUsed"
 	}
-	if fs.GetConfig(ctx).Metadata {
-		fields += "," + metadataFields
-	}
 	return fields
 }
 
 // newRegularObject creates an fs.Object for a normal drive.File
-func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
+func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
 	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
 	if f.opt.SkipChecksumGphotos {
 		for _, space := range info.Spaces {
 			if space == "photos" {
 				info.Md5Checksum = ""
-				info.Sha1Checksum = ""
-				info.Sha256Checksum = ""
 				break
 			}
 		}
 	}
 	o := &Object{
+		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
 		md5sum:     strings.ToLower(info.Md5Checksum),
-		sha1sum:    strings.ToLower(info.Sha1Checksum),
-		sha256sum:  strings.ToLower(info.Sha256Checksum),
 		v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
 	}
-	o.baseObject, err = f.newBaseObject(ctx, remote, info)
-	if err != nil {
-		return nil, err
-	}
 	if info.ResourceKey != "" {
 		o.resourceKey = &info.ResourceKey
 	}
-	return o, nil
+	return o
 }
 
 // newDocumentObject creates an fs.Object for a google docs drive.File
-func (f *Fs) newDocumentObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
+func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	mediaType, _, err := mime.ParseMediaType(exportMimeType)
 	if err != nil {
 		return nil, err
 	}
 	url := info.ExportLinks[mediaType]
-	baseObject, err := f.newBaseObject(ctx, remote+extension, info)
-	if err != nil {
-		return nil, err
-	}
+	baseObject := f.newBaseObject(remote+extension, info)
 	baseObject.bytes = -1
 	baseObject.mimeType = exportMimeType
 	return &documentObject{
@@ -1587,7 +1397,7 @@ func (f *Fs) newDocumentObject(ctx context.Context, remote string, info *drive.F
 }
 
 // newLinkObject creates an fs.Object that represents a link a google docs drive.File
-func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
+func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	t := linkTemplate(exportMimeType)
 	if t == nil {
 		return nil, fmt.Errorf("unsupported link type %s", exportMimeType)
@@ -1606,10 +1416,7 @@ func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File,
 		return nil, fmt.Errorf("executing template failed: %w", err)
 	}
 
-	baseObject, err := f.newBaseObject(ctx, remote+extension, info)
-	if err != nil {
-		return nil, err
-	}
+	baseObject := f.newBaseObject(remote+extension, info)
 	baseObject.bytes = int64(buf.Len())
 	baseObject.mimeType = exportMimeType
 	return &linkObject{
@@ -1625,7 +1432,7 @@ func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File,
 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
 	// If item has MD5 sum it is a file stored on drive
 	if info.Md5Checksum != "" {
-		return f.newRegularObject(ctx, remote, info)
+		return f.newRegularObject(remote, info), nil
 	}
 
 	extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
@@ -1656,15 +1463,13 @@ func (f *Fs) newObjectWithExportInfo(
 	case info.MimeType == shortcutMimeTypeDangling:
 		// Pretend a dangling shortcut is a regular object
 		// It will error if used, but appear in listings so it can be deleted
-		return f.newRegularObject(ctx, remote, info)
+		return f.newRegularObject(remote, info), nil
 	case info.Md5Checksum != "":
 		// If item has MD5 sum it is a file stored on drive
-		return f.newRegularObject(ctx, remote, info)
+		return f.newRegularObject(remote, info), nil
 	case f.opt.SkipGdocs:
 		fs.Debugf(remote, "Skipping google document type %q", info.MimeType)
 		return nil, fs.ErrorObjectNotFound
-	case f.opt.ShowAllGdocs:
-		return f.newDocumentObject(ctx, remote, info, "", info.MimeType)
 	default:
 		// If item MimeType is in the ExportFormats then it is a google doc
 		if !isDocument {
@@ -1676,18 +1481,15 @@ func (f *Fs) newObjectWithExportInfo(
 			return nil, fs.ErrorObjectNotFound
 		}
 		if isLinkMimeType(exportMimeType) {
-			return f.newLinkObject(ctx, remote, info, extension, exportMimeType)
+			return f.newLinkObject(remote, info, extension, exportMimeType)
 		}
-		return f.newDocumentObject(ctx, remote, info, extension, exportMimeType)
+		return f.newDocumentObject(remote, info, extension, exportMimeType)
 	}
 }
 
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	if strings.HasSuffix(remote, "/") {
-		return nil, fs.ErrorIsDir
-	}
 	info, extension, exportName, exportMimeType, isDocument, err := f.getRemoteInfoWithExport(ctx, remote)
 	if err != nil {
 		return nil, err
@@ -2057,7 +1859,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// drive where (A in parents) or (B in parents) returns nothing
 		// sometimes. See #3114, #4289 and
 		// https://issuetracker.google.com/issues/149522397
-		if f.opt.FastListBugFix && len(dirs) > 1 && !foundItems {
+		if len(dirs) > 1 && !foundItems {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
 				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
@@ -2306,7 +2108,7 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
 		fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
 		return item, nil
 	}
-	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.getFileFields(ctx))
+	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
 		var gerr *googleapi.Error
 		if errors.As(err, &gerr) && gerr.Code == 404 {
@@ -2438,10 +2240,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	} else {
 		createInfo.MimeType = fs.MimeTypeFromName(remote)
 	}
-	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, options, createInfo, false)
-	if err != nil {
-		return nil, err
-	}
 
 	var info *drive.File
 	if size >= 0 && size < int64(f.opt.UploadCutoff) {
@@ -2466,10 +2264,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 			return nil, err
 		}
 	}
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
 	return f.newObjectWithInfo(ctx, remote, info)
 }
 
@@ -3083,7 +2877,6 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			if f.rootFolderID == "appDataFolder" {
 				changesCall.Spaces("appDataFolder")
 			}
-			changesCall.RestrictToMyDrive(!f.opt.SharedWithMe)
 			changeList, err = changesCall.Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
@@ -3158,7 +2951,7 @@ func (f *Fs) DirCacheFlush() {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet(hash.MD5, hash.SHA1, hash.SHA256)
+	return hash.Set(hash.MD5)
 }
 
 func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
@@ -3387,7 +3180,7 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 
 // copy file with id to dest
 func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
-	info, err := f.getFile(ctx, id, f.getFileFields(ctx))
+	info, err := f.getFile(ctx, id, f.fileFields)
 	if err != nil {
 		return fmt.Errorf("couldn't find id: %w", err)
 	}
@@ -3530,9 +3323,9 @@ This takes an optional directory to trash which make this easier to
 use via the API.
 
     rclone backend untrash drive:directory
-    rclone backend --interactive untrash drive:directory subdir
+    rclone backend -i untrash drive:directory subdir
 
-Use the --interactive/-i or --dry-run flag to see what would be restored before restoring it.
+Use the -i flag to see what would be restored before restoring it.
 
 Result:
 
@@ -3562,7 +3355,7 @@ component will be used as the file name.
 If the destination is a drive backend then server-side copying will be
 attempted if possible.
 
-Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
+Use the -i flag to see what would be copied before copying.
 `,
 }, {
 	Name:  "exportformats",
@@ -3638,12 +3431,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
+		re := regexp.MustCompile(`[^\w_. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
 			upstreams := []string{}
 			names := make(map[string]struct{}, len(drives))
 			for i, drive := range drives {
-				name := fspath.MakeConfigName(drive.Name)
+				name := re.ReplaceAllString(drive.Name, "_")
 				for {
 					if _, found := names[name]; !found {
 						break
@@ -3719,17 +3513,11 @@ func (o *baseObject) Remote() string {
 
 // Hash returns the Md5sum of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == hash.MD5 {
-		return o.md5sum, nil
-	}
-	if t == hash.SHA1 {
-		return o.sha1sum, nil
-	}
-	if t == hash.SHA256 {
-		return o.sha256sum, nil
-	}
+	if t != hash.MD5 {
 		return "", hash.ErrUnsupported
 	}
+	return o.md5sum, nil
+}
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 {
 		return "", hash.ErrUnsupported
@@ -4067,21 +3855,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		MimeType:     srcMimeType,
 		ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
 	}
-
-	updateMetadata, err := o.fs.fetchAndUpdateMetadata(ctx, src, options, updateInfo, true)
-	if err != nil {
-		return err
-	}
-
 	info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
 	if err != nil {
 		return err
 	}
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return err
-	}
-	newO, err := o.fs.newObjectWithInfo(ctx, o.remote, info)
+	newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
 	if err != nil {
 		return err
 	}
@@ -4166,26 +3944,6 @@ func (o *baseObject) ParentID() string {
 	return ""
 }
 
-// Metadata returns metadata for an object
-//
-// It should return nil if there is no Metadata
-func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
-	if o.metadata != nil {
-		return *o.metadata, nil
-	}
-	fs.Debugf(o, "Fetching metadata")
-	id := actualID(o.id)
-	info, err := o.fs.getFile(ctx, id, o.fs.getFileFields(ctx))
-	if err != nil {
-		return nil, err
-	}
-	err = o.parseMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
-	return *o.metadata, nil
-}
-
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
@@ -4247,7 +4005,6 @@ var (
 	_ fs.MimeTyper       = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 	_ fs.ParentIDer      = (*Object)(nil)
-	_ fs.Metadataer      = (*Object)(nil)
 	_ fs.Object          = (*documentObject)(nil)
 	_ fs.MimeTyper       = (*documentObject)(nil)
 	_ fs.IDer            = (*documentObject)(nil)

backend/drive/drive_internal_test.go

@@ -243,15 +243,6 @@ func (f *Fs) InternalTestShouldRetry(t *testing.T) {
 	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
 	assert.False(t, quotaExceededRetry)
 	assert.Equal(t, quotaExceededError, expectedQuotaError)
-
-	sqEItem := googleapi.ErrorItem{
-		Reason: "storageQuotaExceeded",
-	}
-	generic403.Errors[0] = sqEItem
-	expectedStorageQuotaError := fserrors.FatalError(&generic403)
-	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, storageQuotaExceededRetry)
-	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
 }
 
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {

backend/drive/metadata.go

@@ -1,608 +0,0 @@
-package drive
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/rclone/rclone/fs"
-	"golang.org/x/sync/errgroup"
-	drive "google.golang.org/api/drive/v3"
-	"google.golang.org/api/googleapi"
-)
-
-// system metadata keys which this backend owns
-var systemMetadataInfo = map[string]fs.MetadataHelp{
-	"content-type": {
-		Help:    "The MIME type of the file.",
-		Type:    "string",
-		Example: "text/plain",
-	},
-	"mtime": {
-		Help:    "Time of last modification with mS accuracy.",
-		Type:    "RFC 3339",
-		Example: "2006-01-02T15:04:05.999Z07:00",
-	},
-	"btime": {
-		Help:    "Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates.",
-		Type:    "RFC 3339",
-		Example: "2006-01-02T15:04:05.999Z07:00",
-	},
-	"copy-requires-writer-permission": {
-		Help:    "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.",
-		Type:    "boolean",
-		Example: "true",
-	},
-	"writers-can-share": {
-		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated for items in shared drives.",
-		Type:    "boolean",
-		Example: "false",
-	},
-	"viewed-by-me": {
-		Help:     "Whether the file has been viewed by this user.",
-		Type:     "boolean",
-		Example:  "true",
-		ReadOnly: true,
-	},
-	"owner": {
-		Help:    "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.",
-		Type:    "string",
-		Example: "user@example.com",
-	},
-	"permissions": {
-		Help:    "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.",
-		Type:    "JSON",
-		Example: "{}",
-	},
-	"folder-color-rgb": {
-		Help:    "The color for a folder or a shortcut to a folder as an RGB hex string.",
-		Type:    "string",
-		Example: "881133",
-	},
-	"description": {
-		Help:    "A short description of the file.",
-		Type:    "string",
-		Example: "Contract for signing",
-	},
-	"starred": {
-		Help:    "Whether the user has starred the file.",
-		Type:    "boolean",
-		Example: "false",
-	},
-	"labels": {
-		Help:    "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.",
-		Type:    "JSON",
-		Example: "[]",
-	},
-}
-
-// Extra fields we need to fetch to implement the system metadata above
-var metadataFields = googleapi.Field(strings.Join([]string{
-	"copyRequiresWriterPermission",
-	"description",
-	"folderColorRgb",
-	"hasAugmentedPermissions",
-	"owners",
-	"permissionIds",
-	"permissions",
-	"properties",
-	"starred",
-	"viewedByMe",
-	"viewedByMeTime",
-	"writersCanShare",
-}, ","))
-
-// Fields we need to read from permissions
-var permissionsFields = googleapi.Field(strings.Join([]string{
-	"*",
-	"permissionDetails/*",
-}, ","))
-
-// getPermission returns permissions for the fileID and permissionID passed in
-func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
-	f.permissionsMu.Lock()
-	defer f.permissionsMu.Unlock()
-	if useCache {
-		perm = f.permissions[permissionID]
-		if perm != nil {
-			return perm, false, nil
-		}
-	}
-	fs.Debugf(f, "Fetching permission %q", permissionID)
-	err = f.pacer.Call(func() (bool, error) {
-		perm, err = f.svc.Permissions.Get(fileID, permissionID).
-			Fields(permissionsFields).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, false, err
-	}
-
-	inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited
-
-	cleanPermission(perm)
-
-	// cache the permission
-	f.permissions[permissionID] = perm
-
-	return perm, inherited, err
-}
-
-// Set the permissions on the info
-func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
-	for _, perm := range permissions {
-		if perm.Role == "owner" {
-			// ignore owner permissions - these are set with owner
-			continue
-		}
-		cleanPermissionForWrite(perm)
-		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Permissions.Create(info.Id, perm).
-				SupportsAllDrives(true).
-				Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			return fmt.Errorf("failed to set permission: %w", err)
-		}
-	}
-	return nil
-}
-
-// Clean attributes from permissions which we can't write
-func cleanPermissionForWrite(perm *drive.Permission) {
-	perm.Deleted = false
-	perm.DisplayName = ""
-	perm.Id = ""
-	perm.Kind = ""
-	perm.PermissionDetails = nil
-	perm.TeamDrivePermissionDetails = nil
-}
-
-// Clean and cache the permission if not already cached
-func (f *Fs) cleanAndCachePermission(perm *drive.Permission) {
-	f.permissionsMu.Lock()
-	defer f.permissionsMu.Unlock()
-	cleanPermission(perm)
-	if _, found := f.permissions[perm.Id]; !found {
-		f.permissions[perm.Id] = perm
-	}
-}
-
-// Clean fields we don't need to keep from the permission
-func cleanPermission(perm *drive.Permission) {
-	// DisplayName: Output only. The "pretty" name of the value of the
-	// permission. The following is a list of examples for each type of
-	// permission: * `user` - User's full name, as defined for their Google
-	// account, such as "Joe Smith." * `group` - Name of the Google Group,
-	// such as "The Company Administrators." * `domain` - String domain
-	// name, such as "thecompany.com." * `anyone` - No `displayName` is
-	// present.
-	perm.DisplayName = ""
-
-	// Kind: Output only. Identifies what kind of resource this is. Value:
-	// the fixed string "drive#permission".
-	perm.Kind = ""
-
-	// PermissionDetails: Output only. Details of whether the permissions on
-	// this shared drive item are inherited or directly on this item. This
-	// is an output-only field which is present only for shared drive items.
-	perm.PermissionDetails = nil
-
-	// PhotoLink: Output only. A link to the user's profile photo, if
-	// available.
-	perm.PhotoLink = ""
-
-	// TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use
-	// `permissionDetails` instead.
-	perm.TeamDrivePermissionDetails = nil
-}
-
-// Fields we need to read from labels
-var labelsFields = googleapi.Field(strings.Join([]string{
-	"*",
-}, ","))
-
-// getLabels returns labels for the fileID passed in
-func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) {
-	fs.Debugf(f, "Fetching labels for %q", fileID)
-	listLabels := f.svc.Files.ListLabels(fileID).
-		Fields(labelsFields).
-		Context(ctx)
-	for {
-		var info *drive.LabelList
-		err = f.pacer.Call(func() (bool, error) {
-			info, err = listLabels.Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			return nil, err
-		}
-		labels = append(labels, info.Labels...)
-		if info.NextPageToken == "" {
-			break
-		}
-		listLabels.PageToken(info.NextPageToken)
-	}
-	for _, label := range labels {
-		cleanLabel(label)
-	}
-	return labels, nil
-}
-
-// Set the labels on the info
-func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) {
-	if len(labels) == 0 {
-		return nil
-	}
-	req := drive.ModifyLabelsRequest{}
-	for _, label := range labels {
-		req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{
-			FieldModifications: labelFieldsToFieldModifications(label.Fields),
-			LabelId:            label.Id,
-		})
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Files.ModifyLabels(info.Id, &req).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to set owner: %w", err)
-	}
-	return nil
-}
-
-// Convert label fields into something which can set the fields
-func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) {
-	for id, field := range fields {
-		var emails []string
-		for _, user := range field.User {
-			emails = append(emails, user.EmailAddress)
-		}
-		out = append(out, &drive.LabelFieldModification{
-			// FieldId: The ID of the field to be modified.
-			FieldId: id,
-
-			// SetDateValues: Replaces the value of a dateString Field with these
-			// new values. The string must be in the RFC 3339 full-date format:
-			// YYYY-MM-DD.
-			SetDateValues: field.DateString,
-
-			// SetIntegerValues: Replaces the value of an `integer` field with these
-			// new values.
-			SetIntegerValues: field.Integer,
-
-			// SetSelectionValues: Replaces a `selection` field with these new
-			// values.
-			SetSelectionValues: field.Selection,
-
-			// SetTextValues: Sets the value of a `text` field.
-			SetTextValues: field.Text,
-
-			// SetUserValues: Replaces a `user` field with these new values. The
-			// values must be valid email addresses.
-			SetUserValues: emails,
-		})
-	}
-	return out
-}
-
-// Clean fields we don't need to keep from the label
-func cleanLabel(label *drive.Label) {
-	// Kind: This is always drive#label
-	label.Kind = ""
-
-	for name, field := range label.Fields {
-		// Kind: This is always drive#labelField.
-		field.Kind = ""
-
-		// Note the fields are copies so we need to write them
-		// back to the map
-		label.Fields[name] = field
-	}
-}
-
-// Parse the metadata from drive item
-//
-// It should return nil if there is no Metadata
-func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) {
-	metadata := make(fs.Metadata, 16)
-
-	// Dump user metadata first as it overrides system metadata
-	for k, v := range info.Properties {
-		metadata[k] = v
-	}
-
-	// System metadata
-	metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)
-	metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare)
-	metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe)
-	metadata["content-type"] = info.MimeType
-
-	// Owners: Output only. The owner of this file. Only certain legacy
-	// files may have more than one owner. This field isn't populated for
-	// items in shared drives.
-	if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 {
-		user := info.Owners[0]
-		if len(info.Owners) > 1 {
-			fs.Logf(o, "Ignoring more than 1 owner")
-		}
-		if user != nil {
-			id := user.EmailAddress
-			if id == "" {
-				id = user.DisplayName
-			}
-			metadata["owner"] = id
-		}
-	}
-
-	if o.fs.opt.MetadataPermissions.IsSet(rwRead) {
-		// We only write permissions out if they are not inherited.
-		//
-		// On My Drives permissions seem to be attached to every item
-		// so they will always be written out.
-		//
-		// On Shared Drives only non-inherited permissions will be
-		// written out.
-
-		// To read the inherited permissions flag will mean we need to
-		// read the permissions for each object and the cache will be
-		// useless. However shared drives don't return permissions
-		// only permissionIds so will need to fetch them for each
-		// object. We use HasAugmentedPermissions to see if there are
-		// special permissions before fetching them to save transactions.
-
-		// HasAugmentedPermissions: Output only. Whether there are permissions
-		// directly on this file. This field is only populated for items in
-		// shared drives.
-		if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
-			// Don't process permissions if there aren't any specifically set
-			info.Permissions = nil
-			info.PermissionIds = nil
-		}
-
-		// PermissionIds: Output only. List of permission IDs for users with
-		// access to this file.
-		//
-		// Only process these if we have no Permissions
-		if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 {
-			info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds))
-			g, gCtx := errgroup.WithContext(ctx)
-			g.SetLimit(o.fs.ci.Checkers)
-			var mu sync.Mutex // protect the info.Permissions from concurrent writes
-			for _, permissionID := range info.PermissionIds {
-				permissionID := permissionID
-				g.Go(func() error {
-					// must fetch the team drive ones individually to check the inherited flag
-					perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive)
-					if err != nil {
-						return fmt.Errorf("failed to read permission: %w", err)
-					}
-					// Don't write inherited permissions out
-					if inherited {
-						return nil
-					}
-					// Don't write owner role out - these are covered by the owner metadata
-					if perm.Role == "owner" {
-						return nil
-					}
-					mu.Lock()
-					info.Permissions = append(info.Permissions, perm)
-					mu.Unlock()
-					return nil
-				})
-			}
-			err = g.Wait()
-			if err != nil {
-				return err
-			}
-		} else {
-			// Clean the fetched permissions
-			for _, perm := range info.Permissions {
-				o.fs.cleanAndCachePermission(perm)
-			}
-		}
-
-		// Permissions: Output only. The full list of permissions for the file.
-		// This is only available if the requesting user can share the file. Not
-		// populated for items in shared drives.
-		if len(info.Permissions) > 0 {
-			buf, err := json.Marshal(info.Permissions)
-			if err != nil {
-				return fmt.Errorf("failed to marshal permissions: %w", err)
-			}
-			metadata["permissions"] = string(buf)
-		}
-
-		// Permission propagation
-		// https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation
-		// Leads me to believe that in non shared drives, permissions
-		// are added to each item when you set permissions for a
-		// folder whereas in shared drives they are inherited and
-		// placed on the item directly.
-	}
-
-	if info.FolderColorRgb != "" {
-		metadata["folder-color-rgb"] = info.FolderColorRgb
-	}
-	if info.Description != "" {
-		metadata["description"] = info.Description
-	}
-	metadata["starred"] = fmt.Sprint(info.Starred)
-	metadata["btime"] = info.CreatedTime
-	metadata["mtime"] = info.ModifiedTime
-
-	if o.fs.opt.MetadataLabels.IsSet(rwRead) {
-		// FIXME would be really nice if we knew if files had labels
-		// before listing but we need to know all possible label IDs
-		// to get it in the listing.
-
-		labels, err := o.fs.getLabels(ctx, actualID(info.Id))
-		if err != nil {
-			return fmt.Errorf("failed to fetch labels: %w", err)
-		}
-		buf, err := json.Marshal(labels)
-		if err != nil {
-			return fmt.Errorf("failed to marshal labels: %w", err)
-		}
-		metadata["labels"] = string(buf)
-	}
-
-	o.metadata = &metadata
-	return nil
-}
-
-// Set the owner on the info
-func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) {
-	perm := drive.Permission{
-		Role:         "owner",
-		EmailAddress: owner,
-		// Type: The type of the grantee. Valid values are: * `user` * `group` *
-		// `domain` * `anyone` When creating a permission, if `type` is `user`
-		// or `group`, you must provide an `emailAddress` for the user or group.
-		// When `type` is `domain`, you must provide a `domain`. There isn't
-		// extra information required for an `anyone` type.
-		Type: "user",
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Permissions.Create(info.Id, &perm).
-			SupportsAllDrives(true).
-			TransferOwnership(true).
-			// SendNotificationEmail(false). - required apparently!
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to set owner: %w", err)
-	}
-	return nil
-}
-
-// Call back to set metadata that can't be set on the upload/update
-//
-// The *drive.File passed in holds the current state of the drive.File
-// and this should update it with any modifications.
-type updateMetadataFn func(context.Context, *drive.File) error
-
-// read the metadata from meta and write it into updateInfo
-//
-// update should be true if this is being used to create metadata for
-// an update/PATCH call as the rules on what can be updated are
-// slightly different there.
-//
-// It returns a callback which should be called to finish the updates
-// after the data is uploaded.
-func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) {
-	callbackFns := []updateMetadataFn{}
-	callback = func(ctx context.Context, info *drive.File) error {
-		for _, fn := range callbackFns {
-			err := fn(ctx, info)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	// merge metadata into request and user metadata
-	for k, v := range meta {
-		k, v := k, v
-		// parse a boolean from v and write into out
-		parseBool := func(out *bool) error {
-			b, err := strconv.ParseBool(v)
-			if err != nil {
-				return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err)
-			}
-			*out = b
-			return nil
-		}
-		switch k {
-		case "copy-requires-writer-permission":
-			if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
-				return nil, err
-			}
-		case "writers-can-share":
-			if err := parseBool(&updateInfo.WritersCanShare); err != nil {
-				return nil, err
-			}
-		case "viewed-by-me":
-			// Can't write this
-		case "content-type":
-			updateInfo.MimeType = v
-		case "owner":
-			if !f.opt.MetadataOwner.IsSet(rwWrite) {
-				continue
-			}
-			// Can't set Owner on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				return f.setOwner(ctx, info, v)
-			})
-		case "permissions":
-			if !f.opt.MetadataPermissions.IsSet(rwWrite) {
-				continue
-			}
-			var perms []*drive.Permission
-			err := json.Unmarshal([]byte(v), &perms)
-			if err != nil {
-				return nil, fmt.Errorf("failed to unmarshal permissions: %w", err)
-			}
-			// Can't set Permissions on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				return f.setPermissions(ctx, info, perms)
-			})
-		case "labels":
-			if !f.opt.MetadataLabels.IsSet(rwWrite) {
-				continue
-			}
-			var labels []*drive.Label
-			err := json.Unmarshal([]byte(v), &labels)
-			if err != nil {
-				return nil, fmt.Errorf("failed to unmarshal labels: %w", err)
-			}
-			// Can't set Labels on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				return f.setLabels(ctx, info, labels)
-			})
-		case "folder-color-rgb":
-			updateInfo.FolderColorRgb = v
-		case "description":
-			updateInfo.Description = v
-		case "starred":
-			if err := parseBool(&updateInfo.Starred); err != nil {
-				return nil, err
-			}
-		case "btime":
-			if update {
-				fs.Debugf(f, "Skipping btime metadata as can't update it on an existing file: %v", v)
-			} else {
-				updateInfo.CreatedTime = v
-			}
-		case "mtime":
-			updateInfo.ModifiedTime = v
-		default:
-			if updateInfo.Properties == nil {
-				updateInfo.Properties = make(map[string]string, 1)
-			}
-			updateInfo.Properties[k] = v
-		}
-	}
-	return callback, nil
-}
-
-// Fetch metadata and update updateInfo if --metadata is in use
-func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) {
-	meta, err := fs.GetMetadataOptions(ctx, f, src, options)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
-	}
-	callback, err = f.updateMetadata(ctx, updateInfo, meta, update)
-	if err != nil {
-		return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
-	}
-	return callback, nil
-}

backend/dropbox/batcher.go

@@ -8,19 +8,122 @@ package dropbox
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"sync"
+	"time"
 
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/atexit"
 )
 
+const (
+	maxBatchSize          = 1000                   // max size the batch can be
+	defaultTimeoutSync    = 500 * time.Millisecond // kick off the batch if nothing added for this long (sync)
+	defaultTimeoutAsync   = 10 * time.Second       // kick off the batch if nothing added for this long (ssync)
+	defaultBatchSizeAsync = 100                    // default batch size if async
+)
+
+// batcher holds info about the current items waiting for upload
+type batcher struct {
+	f        *Fs                 // Fs this batch is part of
+	mode     string              // configured batch mode
+	size     int                 // maximum size for batch
+	timeout  time.Duration       // idle timeout for batch
+	async    bool                // whether we are using async batching
+	in       chan batcherRequest // incoming items to batch
+	closed   chan struct{}       // close to indicate batcher shut down
+	atexit   atexit.FnHandle     // atexit handle
+	shutOnce sync.Once           // make sure we shutdown once only
+	wg       sync.WaitGroup      // wait for shutdown
+}
+
+// batcherRequest holds an incoming request with a place for a reply
+type batcherRequest struct {
+	commitInfo *files.UploadSessionFinishArg
+	result     chan<- batcherResponse
+}
+
+// Return true if batcherRequest is the quit request
+func (br *batcherRequest) isQuit() bool {
+	return br.commitInfo == nil
+}
+
+// Send this to get the engine to quit
+var quitRequest = batcherRequest{}
+
+// batcherResponse holds a response to be delivered to clients waiting
+// for a batch to complete.
+type batcherResponse struct {
+	err   error
+	entry *files.FileMetadata
+}
+
+// newBatcher creates a new batcher structure
+func newBatcher(ctx context.Context, f *Fs, mode string, size int, timeout time.Duration) (*batcher, error) {
+	// fs.Debugf(f, "Creating batcher with mode %q, size %d, timeout %v", mode, size, timeout)
+	if size > maxBatchSize || size < 0 {
+		return nil, fmt.Errorf("dropbox: batch size must be < %d and >= 0 - it is currently %d", maxBatchSize, size)
+	}
+
+	async := false
+
+	switch mode {
+	case "sync":
+		if size <= 0 {
+			ci := fs.GetConfig(ctx)
+			size = ci.Transfers
+		}
+		if timeout <= 0 {
+			timeout = defaultTimeoutSync
+		}
+	case "async":
+		if size <= 0 {
+			size = defaultBatchSizeAsync
+		}
+		if timeout <= 0 {
+			timeout = defaultTimeoutAsync
+		}
+		async = true
+	case "off":
+		size = 0
+	default:
+		return nil, fmt.Errorf("dropbox: batch mode must be sync|async|off not %q", mode)
+	}
+
+	b := &batcher{
+		f:       f,
+		mode:    mode,
+		size:    size,
+		timeout: timeout,
+		async:   async,
+		in:      make(chan batcherRequest, size),
+		closed:  make(chan struct{}),
+	}
+	if b.Batching() {
+		b.atexit = atexit.Register(b.Shutdown)
+		b.wg.Add(1)
+		go b.commitLoop(context.Background())
+	}
+	return b, nil
+
+}
+
+// Batching returns true if batching is active
+func (b *batcher) Batching() bool {
+	return b.size > 0
+}
+
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
+func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
 	var arg = &files.UploadSessionFinishBatchArg{
 		Entries: items,
 	}
-	err = f.pacer.Call(func() (bool, error) {
-		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
+	err = b.f.pacer.Call(func() (bool, error) {
+		complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
 		// If error is insufficient space then don't retry
 		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -37,10 +140,66 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 	return complete, nil
 }
 
-// Called by the batcher to commit a batch
-func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []*files.FileMetadata, errors []error) (err error) {
+// finishBatchJobStatus waits for the batch to complete returning completed entries
+func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *files.UploadSessionFinishBatchLaunch) (complete *files.UploadSessionFinishBatchResult, err error) {
+	if launchBatchStatus.AsyncJobId == "" {
+		return nil, errors.New("wait for batch completion: empty job ID")
+	}
+	var batchStatus *files.UploadSessionFinishBatchJobStatus
+	sleepTime := 100 * time.Millisecond
+	const maxSleepTime = 1 * time.Second
+	startTime := time.Now()
+	try := 1
+	for {
+		remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
+		if remaining < 0 {
+			break
+		}
+		err = b.f.pacer.Call(func() (bool, error) {
+			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
+				AsyncJobId: launchBatchStatus.AsyncJobId,
+			})
+			return shouldRetry(ctx, err)
+		})
+		if err != nil {
+			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
+		} else {
+			if batchStatus.Tag == "complete" {
+				fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
+				return batchStatus.Complete, nil
+			}
+			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
+		}
+		time.Sleep(sleepTime)
+		sleepTime *= 2
+		if sleepTime > maxSleepTime {
+			sleepTime = maxSleepTime
+		}
+		try++
+	}
+	if err == nil {
+		err = errors.New("batch didn't complete")
+	}
+	return nil, fmt.Errorf("wait for batch failed after %d tries in %v: %w", try, time.Since(startTime), err)
+}
+
+// commit a batch
+func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []chan<- batcherResponse) (err error) {
+	// If commit fails then signal clients if sync
+	var signalled = b.async
+	defer func() {
+		if err != nil && signalled {
+			// Signal to clients that there was an error
+			for _, result := range results {
+				result <- batcherResponse{err: err}
+			}
+		}
+	}()
+	desc := fmt.Sprintf("%s batch length %d starting with: %s", b.mode, len(items), items[0].Commit.Path)
+	fs.Debugf(b.f, "Committing %s", desc)
+
 	// finalise the batch getting either a result or a job id to poll
-	complete, err := f.finishBatch(ctx, items)
+	complete, err := b.finishBatch(ctx, items)
 	if err != nil {
 		return err
 	}
@@ -51,13 +210,19 @@ func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinish
 		return fmt.Errorf("expecting %d items in batch but got %d", len(results), len(entries))
 	}
 
-	// Format results for return
+	// Report results to clients
+	var (
+		errorTag   = ""
+		errorCount = 0
+	)
 	for i := range results {
 		item := entries[i]
+		resp := batcherResponse{}
 		if item.Tag == "success" {
-			results[i] = item.Success
+			resp.entry = item.Success
 		} else {
-			errorTag := item.Tag
+			errorCount++
+			errorTag = item.Tag
 			if item.Failure != nil {
 				errorTag = item.Failure.Tag
 				if item.Failure.LookupFailed != nil {
@@ -70,9 +235,112 @@ func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinish
 					errorTag += "/" + item.Failure.PropertiesError.Tag
 				}
 			}
-			errors[i] = fmt.Errorf("upload failed: %s", errorTag)
+			resp.err = fmt.Errorf("batch upload failed: %s", errorTag)
+		}
+		if !b.async {
+			results[i] <- resp
+		}
+	}
+	// Show signalled so no need to report error to clients from now on
+	signalled = true
+
+	// Report an error if any failed in the batch
+	if errorTag != "" {
+		return fmt.Errorf("batch had %d errors: last error: %s", errorCount, errorTag)
+	}
+
+	fs.Debugf(b.f, "Committed %s", desc)
+	return nil
+}
+
+// commitLoop runs the commit engine in the background
+func (b *batcher) commitLoop(ctx context.Context) {
+	var (
+		items     []*files.UploadSessionFinishArg // current batch of uncommitted files
+		results   []chan<- batcherResponse        // current batch of clients awaiting results
+		idleTimer = time.NewTimer(b.timeout)
+		commit    = func() {
+			err := b.commitBatch(ctx, items, results)
+			if err != nil {
+				fs.Errorf(b.f, "%s batch commit: failed to commit batch length %d: %v", b.mode, len(items), err)
+			}
+			items, results = nil, nil
+		}
+	)
+	defer b.wg.Done()
+	defer idleTimer.Stop()
+	idleTimer.Stop()
+
+outer:
+	for {
+		select {
+		case req := <-b.in:
+			if req.isQuit() {
+				break outer
+			}
+			items = append(items, req.commitInfo)
+			results = append(results, req.result)
+			idleTimer.Stop()
+			if len(items) >= b.size {
+				commit()
+			} else {
+				idleTimer.Reset(b.timeout)
+			}
+		case <-idleTimer.C:
+			if len(items) > 0 {
+				fs.Debugf(b.f, "Batch idle for %v so committing", b.timeout)
+				commit()
 			}
 		}
 
-	return nil
+	}
+	// commit any remaining items
+	if len(items) > 0 {
+		commit()
+	}
+}
+
+// Shutdown finishes any pending batches then shuts everything down
+//
+// Can be called from atexit handler
+func (b *batcher) Shutdown() {
+	if !b.Batching() {
+		return
+	}
+	b.shutOnce.Do(func() {
+		atexit.Unregister(b.atexit)
+		fs.Infof(b.f, "Committing uploads - please wait...")
+		// show that batcher is shutting down
+		close(b.closed)
+		// quit the commitLoop by sending a quitRequest message
+		//
+		// Note that we don't close b.in because that will
+		// cause write to closed channel in Commit when we are
+		// exiting due to a signal.
+		b.in <- quitRequest
+		b.wg.Wait()
+	})
+}
+
+// Commit commits the file using a batch call, first adding it to the
+// batch and then waiting for the batch to complete in a synchronous
+// way if async is not set.
+func (b *batcher) Commit(ctx context.Context, commitInfo *files.UploadSessionFinishArg) (entry *files.FileMetadata, err error) {
+	select {
+	case <-b.closed:
+		return nil, fserrors.FatalError(errors.New("batcher is shutting down"))
+	default:
+	}
+	fs.Debugf(b.f, "Adding %q to batch", commitInfo.Commit.Path)
+	resp := make(chan batcherResponse, 1)
+	b.in <- batcherRequest{
+		commitInfo: commitInfo,
+		result:     resp,
+	}
+	// If running async then don't wait for the result
+	if b.async {
+		return nil, nil
+	}
+	result := <-resp
+	return result.entry, result.err
 }

backend/dropbox/dropbox.go

@@ -47,7 +47,6 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
@@ -59,7 +58,7 @@ import (
 const (
 	rcloneClientID              = "5jcck7diasz0rqy"
 	rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
-	defaultMinSleep             = fs.Duration(10 * time.Millisecond)
+	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	// Upload chunk size - setting too small makes uploads slow.
@@ -122,14 +121,6 @@ var (
 
 	// Errors
 	errNotSupportedInSharedMode = fserrors.NoRetryError(errors.New("not supported in shared files mode"))
-
-	// Configure the batcher
-	defaultBatcherOptions = batcher.Options{
-		MaxBatchSize:          1000,
-		DefaultTimeoutSync:    500 * time.Millisecond,
-		DefaultTimeoutAsync:   10 * time.Second,
-		DefaultBatchSizeAsync: 100,
-	}
 )
 
 // Gets an oauth config with the right scopes
@@ -161,7 +152,7 @@ func init() {
 				},
 			})
 		},
-		Options: append(append(oauthutil.SharedOptions, []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "chunk_size",
 			Help: fmt.Sprintf(`Upload chunk size (< %v).
 
@@ -193,7 +184,6 @@ v1.55 or later is in use everywhere.
 `,
 			Default:  "",
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "shared_files",
 			Help: `Instructs rclone to work on individual shared files.
@@ -220,9 +210,66 @@ shared folder.`,
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name:     "pacer_min_sleep",
-			Default:  defaultMinSleep,
-			Help:     "Minimum time to sleep between API calls.",
+			Name: "batch_mode",
+			Help: `Upload file batching sync|async|off.
+
+This sets the batch mode used by rclone.
+
+For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)
+
+This has 3 possible values
+
+- off - no batching
+- sync - batch uploads and check completion (default)
+- async - batch upload and don't check completion
+
+Rclone will close any outstanding batches when it exits which may make
+a delay on quit.
+`,
+			Default:  "sync",
+			Advanced: true,
+		}, {
+			Name: "batch_size",
+			Help: `Max number of files in upload batch.
+
+This sets the batch size of files to upload. It has to be less than 1000.
+
+By default this is 0 which means rclone which calculate the batch size
+depending on the setting of batch_mode.
+
+- batch_mode: async - default batch_size is 100
+- batch_mode: sync - default batch_size is the same as --transfers
+- batch_mode: off - not in use
+
+Rclone will close any outstanding batches when it exits which may make
+a delay on quit.
+
+Setting this is a great idea if you are uploading lots of small files
+as it will make them a lot quicker. You can use --transfers 32 to
+maximise throughput.
+`,
+			Default:  0,
+			Advanced: true,
+		}, {
+			Name: "batch_timeout",
+			Help: `Max time to allow an idle upload batch before uploading.
+
+If an upload batch is idle for more than this long then it will be
+uploaded.
+
+The default for this is 0 which means rclone will choose a sensible
+default based on the batch_mode in use.
+
+- batch_mode: async - default batch_timeout is 500ms
+- batch_mode: sync - default batch_timeout is 10s
+- batch_mode: off - not in use
+`,
+			Default:  fs.Duration(0),
+			Advanced: true,
+		}, {
+			Name:     "batch_commit_timeout",
+			Help:     `Max time to wait for a batch to finish committing`,
+			Default:  fs.Duration(10 * time.Minute),
 			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
@@ -237,7 +284,7 @@ shared folder.`,
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8,
-		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
+		}}...),
 	})
 }
 
@@ -250,8 +297,8 @@ type Options struct {
 	BatchMode          string               `config:"batch_mode"`
 	BatchSize          int                  `config:"batch_size"`
 	BatchTimeout       fs.Duration          `config:"batch_timeout"`
+	BatchCommitTimeout fs.Duration          `config:"batch_commit_timeout"`
 	AsyncBatch         bool                 `config:"async_batch"`
-	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -271,7 +318,7 @@ type Fs struct {
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
 	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
-	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
+	batcher        *batcher       // batch builder
 }
 
 // Object describes a dropbox object
@@ -395,13 +442,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:  name,
 		opt:   *opt,
 		ci:    ci,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
-	batcherOptions := defaultBatcherOptions
-	batcherOptions.Mode = f.opt.BatchMode
-	batcherOptions.Size = f.opt.BatchSize
-	batcherOptions.Timeout = time.Duration(f.opt.BatchTimeout)
-	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
+	f.batcher, err = newBatcher(ctx, f, f.opt.BatchMode, f.opt.BatchSize, time.Duration(f.opt.BatchTimeout))
 	if err != nil {
 		return nil, err
 	}
@@ -493,7 +536,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			default:
 				return nil, err
 			}
-			// if the mount failed we have to abort here
+			// if the moint failed we have to abort here
 		}
 		// if the mount succeeded it's now a normal folder in the users root namespace
 		// we disable shared folder mode and proceed normally
@@ -676,7 +719,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
 		}
 		for _, entry := range res.Entries {
 			leaf := f.opt.Enc.ToStandardName(entry.Name)
-			d := fs.NewDir(leaf, time.Time{}).SetID(entry.SharedFolderId)
+			d := fs.NewDir(leaf, time.Now()).SetID(entry.SharedFolderId)
 			entries = append(entries, d)
 			if err != nil {
 				return nil, err
@@ -863,7 +906,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
-				d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
+				d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
 				entries = append(entries, d)
 			} else if fileInfo != nil {
 				o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
@@ -946,7 +989,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	if root == "/" {
 		return errors.New("can't remove root directory")
 	}
-	encRoot := f.opt.Enc.FromStandardPath(root)
 
 	if check {
 		// check directory exists
@@ -955,9 +997,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 			return fmt.Errorf("Rmdir: %w", err)
 		}
 
+		root = f.opt.Enc.FromStandardPath(root)
 		// check directory empty
 		arg := files.ListFolderArg{
-			Path:      encRoot,
+			Path:      root,
 			Recursive: false,
 		}
 		if root == "/" {
@@ -978,7 +1021,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
+		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1231,21 +1274,18 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return nil, err
 	}
 	var total uint64
-	var used = q.Used
 	if q.Allocation != nil {
 		if q.Allocation.Individual != nil {
 			total += q.Allocation.Individual.Allocated
 		}
 		if q.Allocation.Team != nil {
 			total += q.Allocation.Team.Allocated
-			// Override used with Team.Used as this includes q.Used already
-			used = q.Allocation.Team.Used
 		}
 	}
 	usage = &fs.Usage{
 		Total: fs.NewUsageValue(int64(total)),          // quota of bytes that can be used
-		Used:  fs.NewUsageValue(int64(used)),         // bytes in use
-		Free:  fs.NewUsageValue(int64(total - used)), // bytes which can be uploaded before reaching the quota
+		Used:  fs.NewUsageValue(int64(q.Used)),         // bytes in use
+		Free:  fs.NewUsageValue(int64(total - q.Used)), // bytes which can be uploaded before reaching the quota
 	}
 	return usage, nil
 }
@@ -1675,7 +1715,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	// If we are batching then we should have written all the data now
 	// store the commit info now for a batch commit
 	if o.fs.batcher.Batching() {
-		return o.fs.batcher.Commit(ctx, o.remote, args)
+		return o.fs.batcher.Commit(ctx, args)
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/fichier/api.go

@@ -28,14 +28,14 @@ var retryErrorCodes = []int{
 	509, // Bandwidth Limit Exceeded
 }
 
-var errorRegex = regexp.MustCompile(`#(\d{1,3})`)
+var errorRegex = regexp.MustCompile(`#\d{1,3}`)
 
 func parseFichierError(err error) int {
 	matches := errorRegex.FindStringSubmatch(err.Error())
 	if len(matches) == 0 {
 		return 0
 	}
-	code, err := strconv.Atoi(matches[1])
+	code, err := strconv.Atoi(matches[0])
 	if err != nil {
 		fs.Debugf(nil, "failed parsing fichier error: %v", err)
 		return 0
@@ -118,9 +118,6 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
 		Single: 1,
 		Pass:   f.opt.FilePassword,
 	}
-	if f.opt.CDN {
-		request.CDN = 1
-	}
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/download/get_token.cgi",
@@ -408,32 +405,6 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
 	return response, nil
 }
 
-func (f *Fs) moveDir(ctx context.Context, folderID int, newLeaf string, destinationFolderID int) (response *MoveDirResponse, err error) {
-	request := &MoveDirRequest{
-		FolderID:            folderID,
-		DestinationFolderID: destinationFolderID,
-		Rename:              newLeaf,
-		// DestinationUser:     destinationUser,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/mv.cgi",
-	}
-
-	response = &MoveDirResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't move dir: %w", err)
-	}
-
-	return response, nil
-}
-
 func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
 	request := &CopyFileRequest{
 		URLs:     []string{url},
@@ -502,7 +473,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("didn't get an upload node: %w", err)
+		return nil, fmt.Errorf("didnt got an upload node: %w", err)
 	}
 
 	// fs.Debugf(f, "Got Upload node")

backend/fichier/fichier.go

@@ -40,7 +40,6 @@ func init() {
 		Options: []fs.Option{{
 			Help: "Your API Key, get it from https://1fichier.com/console/params.pl.",
 			Name: "api_key",
-			Sensitive: true,
 		}, {
 			Help:     "If you want to download a shared folder, add this parameter.",
 			Name:     "shared_folder",
@@ -55,11 +54,6 @@ func init() {
 			Name:       "folder_password",
 			Advanced:   true,
 			IsPassword: true,
-		}, {
-			Help:     "Set if you wish to use CDN download links.",
-			Name:     "cdn",
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -95,7 +89,6 @@ type Options struct {
 	SharedFolder   string               `config:"shared_folder"`
 	FilePassword   string               `config:"file_password"`
 	FolderPassword string               `config:"folder_password"`
-	CDN            bool                 `config:"cdn"`
 	Enc            encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -340,7 +333,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
 	if size > int64(300e9) {
-		return nil, errors.New("File too big, can't upload")
+		return nil, errors.New("File too big, cant upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles
 	}
@@ -488,51 +481,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
-// DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantDirMove.
-//
-// If destination exists then return fs.ErrorDirExists.
-//
-// This is complicated by the fact that we can't use moveDir to move
-// to a different directory AND rename at the same time as it can
-// overwrite files in the source directory.
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
-	srcFs, ok := src.(*Fs)
-	if !ok {
-		fs.Debugf(srcFs, "Can't move directory - not same remote type")
-		return fs.ErrorCantDirMove
-	}
-
-	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
-	if err != nil {
-		return err
-	}
-	srcIDnumeric, err := strconv.Atoi(srcID)
-	if err != nil {
-		return err
-	}
-	dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID)
-	if err != nil {
-		return err
-	}
-
-	var resp *MoveDirResponse
-	resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric)
-	if err != nil {
-		return fmt.Errorf("couldn't rename leaf: %w", err)
-	}
-	if resp.Status != "OK" {
-		return fmt.Errorf("couldn't rename leaf: %s", resp.Message)
-	}
-
-	srcFs.dirCache.FlushDir(srcRemote)
-	return nil
-}
-
 // Copy src to this remote using server side move operations.
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
@@ -606,7 +554,6 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 var (
 	_ fs.Fs              = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
-	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)

backend/fichier/structs.go

@@ -20,7 +20,6 @@ type DownloadRequest struct {
 	URL    string `json:"url"`
 	Single int    `json:"single"`
 	Pass   string `json:"pass,omitempty"`
-	CDN    int    `json:"cdn,omitempty"`
 }
 
 // RemoveFolderRequest is the request structure of the corresponding request
@@ -70,22 +69,6 @@ type MoveFileResponse struct {
 	URLs    []string `json:"urls"`
 }
 
-// MoveDirRequest is the request structure of the corresponding request
-type MoveDirRequest struct {
-	FolderID            int    `json:"folder_id"`
-	DestinationFolderID int    `json:"destination_folder_id,omitempty"`
-	DestinationUser     string `json:"destination_user"`
-	Rename              string `json:"rename,omitempty"`
-}
-
-// MoveDirResponse is the response structure of the corresponding request
-type MoveDirResponse struct {
-	Status  string `json:"status"`
-	Message string `json:"message"`
-	OldName string `json:"old_name"`
-	NewName string `json:"new_name"`
-}
-
 // CopyFileRequest is the request structure of the corresponding request
 type CopyFileRequest struct {
 	URLs     []string `json:"urls"`

backend/filefabric/filefabric.go

@@ -84,7 +84,6 @@ Leave blank normally.
 
 Fill in to make rclone start with directory of a given ID.
 `,
-			Sensitive: true,
 		}, {
 			Name: "permanent_token",
 			Help: `Permanent Authentication Token.
@@ -98,7 +97,6 @@ These tokens are normally valid for several years.
 
 For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens
 `,
-			Sensitive: true,
 		}, {
 			Name: "token",
 			Help: `Session Token.
@@ -109,7 +107,6 @@ usually valid for 1 hour.
 Don't set this value - rclone will set it automatically.
 `,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "token_expiry",
 			Help: `Token expiry time.
@@ -158,7 +155,7 @@ type Fs struct {
 	tokenMu         sync.Mutex         // hold when reading the token
 	token           string             // current access token
 	tokenExpiry     time.Time          // time the current token expires
-	tokenExpired    atomic.Int32
+	tokenExpired    int32              // read and written with atomic
 	canCopyWithName bool               // set if detected that can use fi_name in copy
 	precision       time.Duration      // precision reported
 }
@@ -243,7 +240,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, st
 		err = status // return the error from the RPC
 		code := status.GetCode()
 		if code == "login_token_expired" {
-			f.tokenExpired.Add(1)
+			atomic.AddInt32(&f.tokenExpired, 1)
 		} else {
 			for _, retryCode := range retryStatusCodes {
 				if code == retryCode.code {
@@ -323,12 +320,12 @@ func (f *Fs) getToken(ctx context.Context) (token string, err error) {
 	var refreshed = false
 	defer func() {
 		if refreshed {
-			f.tokenExpired.Store(0)
+			atomic.StoreInt32(&f.tokenExpired, 0)
 		}
 		f.tokenMu.Unlock()
 	}()
 
-	expired := f.tokenExpired.Load() != 0
+	expired := atomic.LoadInt32(&f.tokenExpired) != 0
 	if expired {
 		fs.Debugf(f, "Token invalid - refreshing")
 	}

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/jlaffaye/ftp"
+	"github.com/rclone/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -28,7 +28,6 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/proxy"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -52,12 +51,10 @@ func init() {
 			Name:     "host",
 			Help:     "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:    "user",
 			Help:    "FTP username.",
 			Default: currentUser,
-			Sensitive: true,
 		}, {
 			Name:    "port",
 			Help:    "FTP port number.",
@@ -73,7 +70,7 @@ func init() {
 When using implicit FTP over TLS the client connects using TLS
 right from the start which breaks compatibility with
 non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTPS.`,
+than port 21. Cannot be used in combination with explicit FTP.`,
 			Default: false,
 		}, {
 			Name: "explicit_tls",
@@ -81,7 +78,7 @@ than port 21. Cannot be used in combination with explicit FTPS.`,
 
 When using explicit FTP over TLS the client explicitly requests
 security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTPS.`,
+to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
@@ -173,18 +170,6 @@ Enabled by default. Use 0 to disable.`,
 			Help: `Allow asking for FTP password when needed.
 
 If this is set and no password is supplied then rclone will ask for a password
-`,
-			Advanced: true,
-		}, {
-			Name:    "socks_proxy",
-			Default: "",
-			Help: `Socks 5 proxy host.
-		
-		Supports the format user:pass@host:port, user@host:port, host:port.
-		
-		Example:
-		
-			myUser:myPass@localhost:9005
 `,
 			Advanced: true,
 		}, {
@@ -231,7 +216,6 @@ type Options struct {
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
-	SocksProxy        string               `config:"socks_proxy"`
 }
 
 // Fs represents a remote FTP server
@@ -249,6 +233,7 @@ type Fs struct {
 	pool     []*ftp.ServerConn
 	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
+	tlsConf  *tls.Config
 	pacer    *fs.Pacer // pacer for FTP connections
 	fGetTime bool      // true if the ftp library accepts GetTime
 	fSetTime bool      // true if the ftp library accepts SetTime
@@ -330,67 +315,26 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-// Return a *textproto.Error if err contains one or nil otherwise
-func textprotoError(err error) (errX *textproto.Error) {
-	if errors.As(err, &errX) {
-		return errX
-	}
-	return nil
-}
-
-// returns true if this FTP error should be retried
-func isRetriableFtpError(err error) bool {
-	if errX := textprotoError(err); errX != nil {
-		switch errX.Code {
-		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
-			return true
-		}
-	}
-	return false
-}
-
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	if isRetriableFtpError(err) {
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusNotAvailable:
 			return true, err
 		}
+	}
 	return fserrors.ShouldRetry(err), err
 }
 
-// Get a TLS config with a unique session cache.
-//
-// We can't share session caches between connections.
-//
-// See: https://github.com/rclone/rclone/issues/7234
-func (f *Fs) tlsConfig() *tls.Config {
-	var tlsConfig *tls.Config
-	if f.opt.TLS || f.opt.ExplicitTLS {
-		tlsConfig = &tls.Config{
-			ServerName:         f.opt.Host,
-			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
-		}
-		if f.opt.TLSCacheSize > 0 {
-			tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(f.opt.TLSCacheSize)
-		}
-		if f.opt.DisableTLS13 {
-			tlsConfig.MaxVersion = tls.VersionTLS12
-		}
-	}
-	return tlsConfig
-}
-
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
-	// tls.Config for this connection only. Will be used for data
-	// and control connections.
-	tlsConfig := f.tlsConfig()
-
 	// Make ftp library dial with fshttp dialer optionally using TLS
 	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
@@ -398,17 +342,12 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		defer func() {
 			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
 		}()
-		baseDialer := fshttp.NewDialer(ctx)
-		if f.opt.SocksProxy != "" {
-			conn, err = proxy.SOCKS5Dial(network, address, f.opt.SocksProxy, baseDialer)
-		} else {
-			conn, err = baseDialer.Dial(network, address)
-		}
+		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
 		if err != nil {
 			return nil, err
 		}
 		// Connect using cleartext only for non TLS
-		if tlsConfig == nil {
+		if f.tlsConf == nil {
 			return conn, nil
 		}
 		// Initial connection only needs to be cleartext for explicit TLS
@@ -417,7 +356,7 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 			return conn, nil
 		}
 		// Upgrade connection to TLS
-		tlsConn := tls.Client(conn, tlsConfig)
+		tlsConn := tls.Client(conn, f.tlsConf)
 		// Do the initial handshake - tls.Client doesn't do it for us
 		// If we do this then connections to proftpd/pureftpd lock up
 		// See: https://github.com/rclone/rclone/issues/6426
@@ -439,9 +378,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
 		// as a trigger for sending PSBZ and PROT options to server.
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
+		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
-		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -524,7 +463,8 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
-		if tpErr := textprotoError(err); tpErr != nil {
+		var tpErr *textproto.Error
+		if !errors.As(err, &tpErr) {
 			nopErr := c.NoOp()
 			if nopErr != nil {
 				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
@@ -596,6 +536,19 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	if opt.TLS && opt.ExplicitTLS {
 		return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config")
 	}
+	var tlsConfig *tls.Config
+	if opt.TLS || opt.ExplicitTLS {
+		tlsConfig = &tls.Config{
+			ServerName:         opt.Host,
+			InsecureSkipVerify: opt.SkipVerifyTLSCert,
+		}
+		if opt.TLSCacheSize > 0 {
+			tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(opt.TLSCacheSize)
+		}
+		if opt.DisableTLS13 {
+			tlsConfig.MaxVersion = tls.VersionTLS12
+		}
+	}
 	u := protocol + path.Join(dialAddr+"/", root)
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
@@ -608,11 +561,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		pass:     pass,
 		dialAddr: dialAddr,
 		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
+		tlsConf:  tlsConfig,
 		pacer:    fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
-		PartialUploads:          true,
 	}).Fill(ctx, f)
 	// set the pool drainer timer going
 	if f.opt.IdleTimeout > 0 {
@@ -660,7 +613,8 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
@@ -671,7 +625,8 @@ func translateErrorFile(err error) error {
 
 // translateErrorDir turns FTP errors into rclone errors if possible for a directory
 func translateErrorDir(err error) error {
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
@@ -702,7 +657,8 @@ func (f *Fs) dirFromStandardPath(dir string) string {
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	if remote == "" || remote == "." || remote == "/" {
+	fullPath := path.Join(f.root, remote)
+	if fullPath == "" || fullPath == "." || fullPath == "/" {
 		// if root, assume exists and synthesize an entry
 		return &ftp.Entry{
 			Name: "",
@@ -710,38 +666,13 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 			Time: time.Now(),
 		}, nil
 	}
+	dir := path.Dir(fullPath)
+	base := path.Base(fullPath)
 
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("findItem: %w", err)
 	}
-
-	// returns TRUE if MLST is supported which is required to call GetEntry
-	if c.IsTimePreciseInList() {
-		entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
-		f.putFtpConnection(&c, err)
-		if err != nil {
-			err = translateErrorFile(err)
-			if err == fs.ErrorObjectNotFound {
-				return nil, nil
-			}
-			if errX := textprotoError(err); errX != nil {
-				switch errX.Code {
-				case ftp.StatusBadArguments:
-					err = nil
-				}
-			}
-			return nil, err
-		}
-		if entry != nil {
-			f.entryToStandard(entry)
-		}
-		return entry, nil
-	}
-
-	dir := path.Dir(remote)
-	base := path.Base(remote)
-
 	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -760,7 +691,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, path.Join(f.root, remote))
+	entry, err := f.findItem(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
@@ -782,7 +713,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 
 // dirExists checks the directory pointed to by remote exists or not
 func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, path.Join(f.root, remote))
+	entry, err := f.findItem(ctx, remote)
 	if err != nil {
 		return false, fmt.Errorf("dirExists: %w", err)
 	}
@@ -926,10 +857,23 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // getInfo reads the FileInfo for a path
 func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	file, err := f.findItem(ctx, remote)
+	dir := path.Dir(remote)
+	base := path.Base(remote)
+
+	c, err := f.getFtpConnection(ctx)
 	if err != nil {
-		return nil, err
-	} else if file != nil {
+		return nil, fmt.Errorf("getInfo: %w", err)
+	}
+	files, err := c.List(f.dirFromStandardPath(dir))
+	f.putFtpConnection(&c, err)
+	if err != nil {
+		return nil, translateErrorFile(err)
+	}
+
+	for i := range files {
+		file := files[i]
+		f.entryToStandard(file)
+		if file.Name == base {
 			info := &FileInfo{
 				Name:    remote,
 				Size:    file.Size,
@@ -939,6 +883,7 @@ func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err erro
 			}
 			return info, nil
 		}
+	}
 	return nil, fs.ErrorObjectNotFound
 }
 
@@ -968,7 +913,8 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	}
 	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
@@ -1137,7 +1083,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 // SetModTime sets the modification time of the object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if !o.fs.fSetTime {
-		fs.Debugf(o.fs, "SetModTime is not supported")
+		fs.Errorf(o.fs, "SetModTime is not supported")
 		return nil
 	}
 	c, err := o.fs.getFtpConnection(ctx)
@@ -1209,7 +1155,8 @@ func (f *ftpReadCloser) Close() error {
 	// mask the error if it was caused by a premature close
 	// NB StatusAboutToSend is to work around a bug in pureftpd
 	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
@@ -1235,26 +1182,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-
-	var (
-		fd *ftp.Response
-		c  *ftp.ServerConn
-	)
-	err = o.fs.pacer.Call(func() (bool, error) {
-		c, err = o.fs.getFtpConnection(ctx)
-		if err != nil {
-			return false, err // getFtpConnection has retries already
-		}
-		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-		if err != nil {
-			o.fs.putFtpConnection(&c, err)
-		}
-		return shouldRetry(ctx, err)
-	})
+	c, err := o.fs.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-
+	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+	if err != nil {
+		o.fs.putFtpConnection(&c, err)
+		return nil, fmt.Errorf("open: %w", err)
+	}
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }
@@ -1287,12 +1223,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	// Ignore error 250 here - send by some servers
-	if errX := textprotoError(err); errX != nil {
+	if err != nil {
+		switch errX := err.(type) {
+		case *textproto.Error:
 			switch errX.Code {
 			case ftp.StatusRequestedFileActionOK:
 				err = nil
 			}
 		}
+	}
 	if err != nil {
 		_ = c.Quit() // toss this connection to avoid sync errors
 		// recycle connection in advance to let remove() find free token

backend/googlecloudstorage/googlecloudstorage.go

@@ -82,8 +82,7 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			envAuth, _ := m.Get("env_auth")
-			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
+			if saFile != "" || saCreds != "" || anonymous == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -93,11 +92,6 @@ func init() {
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "project_number",
 			Help: "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
-			Sensitive: true,
-		}, {
-			Name:      "user_project",
-			Help:      "User project.\n\nOptional - needed only for requester pays.",
-			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
@@ -105,7 +99,6 @@ func init() {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
 			Hide: fs.OptionHideBoth,
-			Sensitive: true,
 		}, {
 			Name:    "anonymous",
 			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",
@@ -304,15 +297,6 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
 			}},
-		}, {
-			Name:     "directory_markers",
-			Default:  false,
-			Advanced: true,
-			Help: `Upload an empty object with a trailing slash when a new directory is created
-
-Empty folders are unsupported for bucket based remotes, this option creates an empty
-object ending with "/", to persist the folder.
-`,
 		}, {
 			Name: "no_check_bucket",
 			Help: `If set, don't attempt to check the bucket exists or create it.
@@ -346,17 +330,6 @@ can't check the size and hash but the file contents will be decompressed.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
-		}, {
-			Name:    "env_auth",
-			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
-			Default: false,
-			Examples: []fs.OptionExample{{
-				Value: "false",
-				Help:  "Enter credentials in the next step.",
-			}, {
-				Value: "true",
-				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
-			}},
 		}}...),
 	})
 }
@@ -364,7 +337,6 @@ can't check the size and hash but the file contents will be decompressed.
 // Options defines the configuration for this backend
 type Options struct {
 	ProjectNumber             string               `config:"project_number"`
-	UserProject               string               `config:"user_project"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
 	Anonymous                 bool                 `config:"anonymous"`
@@ -377,8 +349,6 @@ type Options struct {
 	Decompress                bool                 `config:"decompress"`
 	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EnvAuth                   bool                 `config:"env_auth"`
-	DirectoryMarkers          bool                 `config:"directory_markers"`
 }
 
 // Fs represents a remote storage server
@@ -474,7 +444,7 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -530,11 +500,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
-	} else if opt.EnvAuth {
-		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
-		if err != nil {
-			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
-		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {
@@ -560,9 +525,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		BucketBased:       true,
 		BucketBasedRootOK: true,
 	}).Fill(ctx, f)
-	if opt.DirectoryMarkers {
-		f.features.CanHaveEmptyDirectories = true
-	}
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
@@ -579,11 +541,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// Check to see if the object exists
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			get := f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx)
-			if f.opt.UserProject != "" {
-				get = get.UserProject(f.opt.UserProject)
-			}
-			_, err = get.Do()
+			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -643,13 +601,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		directory += "/"
 	}
 	list := f.svc.Objects.List(bucket).Prefix(directory).MaxResults(listChunks)
-	if f.opt.UserProject != "" {
-		list = list.UserProject(f.opt.UserProject)
-	}
 	if !recurse {
 		list = list.Delimiter("/")
 	}
-	foundItems := 0
 	for {
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
@@ -665,7 +619,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			return err
 		}
 		if !recurse {
-			foundItems += len(objects.Prefixes)
 			var object storage.Object
 			for _, remote := range objects.Prefixes {
 				if !strings.HasSuffix(remote, "/") {
@@ -686,29 +639,22 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				}
 			}
 		}
-		foundItems += len(objects.Items)
 		for _, object := range objects.Items {
 			remote := f.opt.Enc.ToStandardPath(object.Name)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", object.Name)
 				continue
 			}
-			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
-			// is this a directory marker?
-			if isDirectory {
-				// Don't insert the root directory
-				if remote == directory {
-					continue
-				}
-				// process directory markers as directories
-				remote = strings.TrimRight(remote, "/")
-			}
 			remote = remote[len(prefix):]
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
-
-			err = fn(remote, object, isDirectory)
+			// is this a directory marker?
+			if isDirectory {
+				continue // skip directory marker
+			}
+			err = fn(remote, object, false)
 			if err != nil {
 				return err
 			}
@@ -718,17 +664,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 		list.PageToken(objects.NextPageToken)
 	}
-	if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
-		// Determine whether the directory exists or not by whether it has a marker
-		_, err := f.readObjectInfo(ctx, bucket, directory)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				return fs.ErrorDirNotFound
-			}
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -772,9 +707,6 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		return nil, errors.New("can't list buckets without project number")
 	}
 	listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks)
-	if f.opt.UserProject != "" {
-		listBuckets = listBuckets.UserProject(f.opt.UserProject)
-	}
 	for {
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
@@ -892,69 +824,10 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	return f.Put(ctx, in, src, options...)
 }
 
-// Create directory marker file and parents
-func (f *Fs) createDirectoryMarker(ctx context.Context, bucket, dir string) error {
-	if !f.opt.DirectoryMarkers || bucket == "" {
-		return nil
-	}
-
-	// Object to be uploaded
-	o := &Object{
-		fs:      f,
-		modTime: time.Now(),
-	}
-
-	for {
-		_, bucketPath := f.split(dir)
-		// Don't create the directory marker if it is the bucket or at the very root
-		if bucketPath == "" {
-			break
-		}
-		o.remote = dir + "/"
-
-		// Check to see if object already exists
-		_, err := o.readObjectInfo(ctx)
-		if err == nil {
-			return nil
-		}
-
-		// Upload it if not
-		fs.Debugf(o, "Creating directory marker")
-		content := io.Reader(strings.NewReader(""))
-		err = o.Update(ctx, content, o)
-		if err != nil {
-			return fmt.Errorf("creating directory marker failed: %w", err)
-		}
-
-		// Now check parent directory exists
-		dir = path.Dir(dir)
-		if dir == "/" || dir == "." {
-			break
-		}
-	}
-
-	return nil
-}
-
 // Mkdir creates the bucket if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 	bucket, _ := f.split(dir)
-	e := f.checkBucket(ctx, bucket)
-	if e != nil {
-		return e
-	}
-	return f.createDirectoryMarker(ctx, bucket, dir)
-
-}
-
-// mkdirParent creates the parent bucket/directory if it doesn't exist
-func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote = strings.TrimRight(remote, "/")
-	dir := path.Dir(remote)
-	if dir == "/" || dir == "." {
-		dir = ""
-	}
-	return f.Mkdir(ctx, dir)
+	return f.makeBucket(ctx, bucket)
 }
 
 // makeBucket creates the bucket if it doesn't exist
@@ -963,11 +836,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
-			list := f.svc.Objects.List(bucket).MaxResults(1).Context(ctx)
-			if f.opt.UserProject != "" {
-				list = list.UserProject(f.opt.UserProject)
-			}
-			_, err = list.Do()
+			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -1002,11 +871,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 			if !f.opt.BucketPolicyOnly {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
-			insertBucket = insertBucket.Context(ctx)
-			if f.opt.UserProject != "" {
-				insertBucket = insertBucket.UserProject(f.opt.UserProject)
-			}
-			_, err = insertBucket.Do()
+			_, err = insertBucket.Context(ctx).Do()
 			return shouldRetry(ctx, err)
 		})
 	}, nil)
@@ -1026,28 +891,12 @@ func (f *Fs) checkBucket(ctx context.Context, bucket string) error {
 // to delete was not empty.
 func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	bucket, directory := f.split(dir)
-	// Remove directory marker file
-	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
-		o := &Object{
-			fs:     f,
-			remote: dir + "/",
-		}
-		fs.Debugf(o, "Removing directory marker")
-		err := o.Remove(ctx)
-		if err != nil {
-			return fmt.Errorf("removing directory marker failed: %w", err)
-		}
-	}
 	if bucket == "" || directory != "" {
 		return nil
 	}
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
-			deleteBucket := f.svc.Buckets.Delete(bucket).Context(ctx)
-			if f.opt.UserProject != "" {
-				deleteBucket = deleteBucket.UserProject(f.opt.UserProject)
-			}
-			err = deleteBucket.Do()
+			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
 			return shouldRetry(ctx, err)
 		})
 	})
@@ -1069,7 +918,7 @@ func (f *Fs) Precision() time.Duration {
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	dstBucket, dstPath := f.split(remote)
-	err := f.mkdirParent(ctx, remote)
+	err := f.checkBucket(ctx, dstBucket)
 	if err != nil {
 		return nil, err
 	}
@@ -1093,11 +942,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var rewriteResponse *storage.RewriteResponse
 	for {
 		err = f.pacer.Call(func() (bool, error) {
-			rewriteRequest = rewriteRequest.Context(ctx)
-			if f.opt.UserProject != "" {
-				rewriteRequest.UserProject(f.opt.UserProject)
-			}
-			rewriteResponse, err = rewriteRequest.Do()
+			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1207,17 +1052,8 @@ func (o *Object) setMetaData(info *storage.Object) {
 // readObjectInfo reads the definition for an object
 func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, err error) {
 	bucket, bucketPath := o.split()
-	return o.fs.readObjectInfo(ctx, bucket, bucketPath)
-}
-
-// readObjectInfo reads the definition for an object
-func (f *Fs) readObjectInfo(ctx context.Context, bucket, bucketPath string) (object *storage.Object, err error) {
-	err = f.pacer.Call(func() (bool, error) {
-		get := f.svc.Objects.Get(bucket, bucketPath).Context(ctx)
-		if f.opt.UserProject != "" {
-			get = get.UserProject(f.opt.UserProject)
-		}
-		object, err = get.Do()
+	err = o.fs.pacer.Call(func() (bool, error) {
+		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1289,11 +1125,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 		if !o.fs.opt.BucketPolicyOnly {
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		copyObject = copyObject.Context(ctx)
-		if o.fs.opt.UserProject != "" {
-			copyObject = copyObject.UserProject(o.fs.opt.UserProject)
-		}
-		newObject, err = copyObject.Do()
+		newObject, err = copyObject.Context(ctx).Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1310,11 +1142,7 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	url := o.url
-	if o.fs.opt.UserProject != "" {
-		url += "&userProject=" + o.fs.opt.UserProject
-	}
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -1357,15 +1185,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
-	// Create parent dir/bucket if not saving directory marker
-	if !strings.HasSuffix(o.remote, "/") {
-		err = o.fs.mkdirParent(ctx, o.remote)
+	err := o.fs.checkBucket(ctx, bucket)
 	if err != nil {
 		return err
 	}
-	}
 	modTime := src.ModTime(ctx)
 
 	object := storage.Object{
@@ -1409,11 +1234,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if !o.fs.opt.BucketPolicyOnly {
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		insertObject = insertObject.Context(ctx)
-		if o.fs.opt.UserProject != "" {
-			insertObject = insertObject.UserProject(o.fs.opt.UserProject)
-		}
-		newObject, err = insertObject.Do()
+		newObject, err = insertObject.Context(ctx).Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1428,11 +1249,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		deleteBucket := o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx)
-		if o.fs.opt.UserProject != "" {
-			deleteBucket = deleteBucket.UserProject(o.fs.opt.UserProject)
-		}
-		err = deleteBucket.Do()
+		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
 		return shouldRetry(ctx, err)
 	})
 	return err

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/googlecloudstorage"
-	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -17,17 +16,3 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*googlecloudstorage.Object)(nil),
 	})
 }
-
-func TestIntegration2(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	name := "TestGoogleCloudStorage"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*googlecloudstorage.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "directory_markers", Value: "true"},
-		},
-	})
-}

backend/googlephotos/googlephotos.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
-	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
@@ -72,14 +71,6 @@ var (
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
 	}
-
-	// Configure the batcher
-	defaultBatcherOptions = batcher.Options{
-		MaxBatchSize:          50,
-		DefaultTimeoutSync:    1000 * time.Millisecond,
-		DefaultTimeoutAsync:   10 * time.Second,
-		DefaultBatchSizeAsync: 50,
-	}
 )
 
 // Register with Fs
@@ -120,7 +111,7 @@ will count towards storage in your Google Account.`)
 			}
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
-		Options: append(append(oauthutil.SharedOptions, []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "read_only",
 			Default: false,
 			Help: `Set to make the Google Photos backend read only.
@@ -167,7 +158,7 @@ listings and won't be transferred.`,
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
-		}}...), defaultBatcherOptions.FsOptions("")...),
+		}}...),
 	})
 }
 
@@ -178,9 +169,6 @@ type Options struct {
 	StartYear       int                  `config:"start_year"`
 	IncludeArchived bool                 `config:"include_archived"`
 	Enc             encoder.MultiEncoder `config:"encoding"`
-	BatchMode       string               `config:"batch_mode"`
-	BatchSize       int                  `config:"batch_size"`
-	BatchTimeout    fs.Duration          `config:"batch_timeout"`
 }
 
 // Fs represents a remote storage server
@@ -199,7 +187,6 @@ type Fs struct {
 	uploadedMu sync.Mutex             // to protect the below
 	uploaded   dirtree.DirTree        // record of uploaded items
 	createMu   sync.Mutex             // held when creating albums to prevent dupes
-	batcher    *batcher.Batcher[uploadedItem, *api.MediaItem]
 }
 
 // Object describes a storage object
@@ -325,14 +312,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		albums:    map[bool]*albums{},
 		uploaded:  dirtree.New(),
 	}
-	batcherOptions := defaultBatcherOptions
-	batcherOptions.Mode = f.opt.BatchMode
-	batcherOptions.Size = f.opt.BatchSize
-	batcherOptions.Timeout = time.Duration(f.opt.BatchTimeout)
-	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
-	if err != nil {
-		return nil, err
-	}
 	f.features = (&fs.Features{
 		ReadMimeType: true,
 	}).Fill(ctx, f)
@@ -802,13 +781,6 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.None)
 }
 
-// Shutdown the backend, closing any background tasks and any
-// cached connections.
-func (f *Fs) Shutdown(ctx context.Context) error {
-	f.batcher.Shutdown()
-	return nil
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -989,82 +961,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return resp.Body, err
 }
 
-// input to the batcher
-type uploadedItem struct {
-	AlbumID     string // desired album
-	UploadToken string // upload ID
-}
-
-// Commit a batch of items to albumID returning the errors in errors
-func (f *Fs) commitBatchAlbumID(ctx context.Context, items []uploadedItem, results []*api.MediaItem, errors []error, albumID string) {
-	// Create the media item from an UploadToken, optionally adding to an album
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/mediaItems:batchCreate",
-	}
-	var request = api.BatchCreateRequest{
-		AlbumID: albumID,
-	}
-	itemsInBatch := 0
-	for i := range items {
-		if items[i].AlbumID == albumID {
-			request.NewMediaItems = append(request.NewMediaItems, api.NewMediaItem{
-				SimpleMediaItem: api.SimpleMediaItem{
-					UploadToken: items[i].UploadToken,
-				},
-			})
-			itemsInBatch++
-		}
-	}
-	var result api.BatchCreateResponse
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		err = fmt.Errorf("failed to create media item: %w", err)
-	}
-	if err == nil && len(result.NewMediaItemResults) != itemsInBatch {
-		err = fmt.Errorf("bad response to BatchCreate expecting %d items but got %d", itemsInBatch, len(result.NewMediaItemResults))
-	}
-	j := 0
-	for i := range items {
-		if items[i].AlbumID == albumID {
-			if err == nil {
-				media := &result.NewMediaItemResults[j]
-				if media.Status.Code != 0 {
-					errors[i] = fmt.Errorf("upload failed: %s (%d)", media.Status.Message, media.Status.Code)
-				} else {
-					results[i] = &media.MediaItem
-				}
-			} else {
-				errors[i] = err
-			}
-			j++
-		}
-	}
-}
-
-// Called by the batcher to commit a batch
-func (f *Fs) commitBatch(ctx context.Context, items []uploadedItem, results []*api.MediaItem, errors []error) (err error) {
-	// Discover all the AlbumIDs as we have to upload these separately
-	//
-	// Should maybe have one batcher per AlbumID
-	albumIDs := map[string]struct{}{}
-	for i := range items {
-		albumIDs[items[i].AlbumID] = struct{}{}
-	}
-
-	// batch the albums
-	for albumID := range albumIDs {
-		// errors returned in errors
-		f.commitBatchAlbumID(ctx, items, results, errors, albumID)
-	}
-	return nil
-}
-
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
@@ -1125,29 +1021,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.New("empty upload token")
 	}
 
-	uploaded := uploadedItem{
+	// Create the media item from an UploadToken, optionally adding to an album
+	opts = rest.Opts{
+		Method: "POST",
+		Path:   "/mediaItems:batchCreate",
+	}
+	var request = api.BatchCreateRequest{
 		AlbumID: albumID,
+		NewMediaItems: []api.NewMediaItem{
+			{
+				SimpleMediaItem: api.SimpleMediaItem{
 					UploadToken: uploadToken,
+				},
+			},
+		},
 	}
-
-	// Save the upload into an album
-	var info *api.MediaItem
-	if o.fs.batcher.Batching() {
-		info, err = o.fs.batcher.Commit(ctx, o.remote, uploaded)
-	} else {
-		errors := make([]error, 1)
-		results := make([]*api.MediaItem, 1)
-		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
+	var result api.BatchCreateResponse
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
 	if err != nil {
-			err = errors[0]
-			info = results[0]
+		return fmt.Errorf("failed to create media item: %w", err)
 	}
+	if len(result.NewMediaItemResults) != 1 {
+		return errors.New("bad response to BatchCreate wrong number of items")
 	}
-	if err != nil {
-		return fmt.Errorf("failed to commit batch: %w", err)
+	mediaItemResult := result.NewMediaItemResults[0]
+	if mediaItemResult.Status.Code != 0 {
+		return fmt.Errorf("upload failed: %s (%d)", mediaItemResult.Status.Message, mediaItemResult.Status.Code)
 	}
-
-	o.setMetaData(info)
+	o.setMetaData(&mediaItemResult.MediaItem)
 
 	// Add upload to internal storage
 	if pattern.isUpload {

backend/googlephotos/googlephotos_test.go

@@ -12,6 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -55,7 +56,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
+			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -220,7 +221,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
+		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/hasher/commands.go

@@ -80,14 +80,6 @@ func (f *Fs) dbDump(ctx context.Context, full bool, root string) error {
 		}
 		root = fspath.JoinRootPath(remoteFs.Root(), f.Root())
 	}
-	if f.db == nil {
-		if f.opt.MaxAge == 0 {
-			fs.Errorf(f, "db not found. (disabled with max_age = 0)")
-		} else {
-			fs.Errorf(f, "db not found.")
-		}
-		return kv.ErrInactive
-	}
 	op := &kvDump{
 		full: full,
 		root: root,
@@ -169,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
 			if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
 				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
 			}
-			accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
+			accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
 			doneCount++
 		}
 	})

backend/hasher/hasher.go

@@ -114,13 +114,6 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 		root: rpath,
 		opt:  opt,
 	}
-	// Correct root if definitely pointing to a file
-	if err == fs.ErrorIsFile {
-		f.root = path.Dir(f.root)
-		if f.root == "." || f.root == "/" {
-			f.root = ""
-		}
-	}
 	baseFeatures := baseFs.Features()
 	f.fpTime = baseFs.Precision() != fs.ModTimeNotSupported
 
@@ -173,7 +166,6 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            true,
-		PartialUploads:          true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -418,9 +410,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Shutdown the backend, closing any background tasks and any cached connections.
 func (f *Fs) Shutdown(ctx context.Context) (err error) {
-	if f.db != nil {
 	err = f.db.Stop(false)
-	}
 	if do := f.Fs.Features().Shutdown; do != nil {
 		if err2 := do(ctx); err2 != nil {
 			err = err2

backend/hasher/hasher_internal_test.go

@@ -60,11 +60,9 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
 	assert.NotNil(t, dst)
 
 	// check that hash was created
-	if f.opt.MaxAge > 0 {
 	hash, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, hash)
-	}
 	//t.Logf("hash is %q", hash)
 	_ = operations.Purge(ctx, f, dirName)
 }

backend/hasher/hasher_test.go

@@ -23,7 +23,6 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*hasher.Object)(nil),
 		UnimplementableFsMethods: []string{
 			"OpenWriterAt",
-			"OpenChunkWriter",
 		},
 		UnimplementableObjectMethods: []string{},
 	}
@@ -37,9 +36,4 @@ func TestIntegration(t *testing.T) {
 		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
-	// test again with MaxAge = 0
-	if *fstest.RemoteName == "" {
-		opt.ExtraConfig = append(opt.ExtraConfig, fstests.ExtraConfigItem{Name: "TestHasher", Key: "max_age", Value: "0"})
-		fstests.Run(t, &opt)
-	}
 }

backend/hdfs/fs.go

@@ -21,7 +21,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/pacer"
 )
 
 // Fs represents a HDFS server
@@ -32,15 +31,8 @@ type Fs struct {
 	opt      Options        // options for this backend
 	ci       *fs.ConfigInfo // global config
 	client   *hdfs.Client
-	pacer    *fs.Pacer // pacer for API calls
 }
 
-const (
-	minSleep      = 20 * time.Millisecond
-	maxSleep      = 10 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
-)
-
 // copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
 func getKerberosClient() (*krb.Client, error) {
 	configPath := os.Getenv("KRB5_CONFIG")
@@ -93,7 +85,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	options := hdfs.ClientOptions{
-		Addresses:           opt.Namenode,
+		Addresses:           []string{opt.Namenode},
 		UseDatanodeHostname: false,
 	}
 
@@ -122,7 +114,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:    *opt,
 		ci:     fs.GetConfig(ctx),
 		client: client,
-		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 
 	f.features = (&fs.Features{

backend/hdfs/hdfs.go

@@ -20,10 +20,8 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "namenode",
-			Help:      "Hadoop name nodes and ports.\n\nE.g. \"namenode-1:8020,namenode-2:8020,...\" to connect to host namenodes at port 8020.",
+			Help:     "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
 			Required: true,
-			Sensitive: true,
-			Default:   fs.CommaSepList{},
 		}, {
 			Name: "username",
 			Help: "Hadoop user name.",
@@ -31,7 +29,6 @@ func init() {
 				Value: "root",
 				Help:  "Connect to hdfs as root.",
 			}},
-			Sensitive: true,
 		}, {
 			Name: "service_principal_name",
 			Help: `Kerberos service principal name for the namenode.
@@ -40,15 +37,14 @@ Enables KERBEROS authentication. Specifies the Service Principal Name
 (SERVICE/FQDN) for the namenode. E.g. \"hdfs/namenode.hadoop.docker\"
 for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "data_transfer_protection",
 			Help: `Kerberos data transfer protection: authentication|integrity|privacy.
 
 Specifies whether or not authentication, data signature integrity
-checks, and wire encryption are required when communicating with
-the datanodes. Possible values are 'authentication', 'integrity'
-and 'privacy'. Used only with KERBEROS enabled.`,
+checks, and wire encryption is required when communicating the the
+datanodes. Possible values are 'authentication', 'integrity' and
+'privacy'. Used only with KERBEROS enabled.`,
 			Examples: []fs.OptionExample{{
 				Value: "privacy",
 				Help:  "Ensure authentication, integrity and encryption enabled.",
@@ -66,7 +62,7 @@ and 'privacy'. Used only with KERBEROS enabled.`,
 
 // Options for this backend
 type Options struct {
-	Namenode               fs.CommaSepList      `config:"namenode"`
+	Namenode               string               `config:"namenode"`
 	Username               string               `config:"username"`
 	ServicePrincipalName   string               `config:"service_principal_name"`
 	DataTransferProtection string               `config:"data_transfer_protection"`

backend/hdfs/object.go

@@ -5,12 +5,10 @@ package hdfs
 
 import (
 	"context"
-	"errors"
 	"io"
 	"path"
 	"time"
 
-	"github.com/colinmarc/hdfs/v2"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/readers"
@@ -108,7 +106,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update object
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	realpath := o.fs.realpath(o.remote)
+	realpath := o.fs.realpath(src.Remote())
 	dirname := path.Dir(realpath)
 	fs.Debugf(o.fs, "update [%s]", realpath)
 
@@ -143,23 +141,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If the datanodes have acknowledged all writes but not yet
-	// to the namenode, FileWriter.Close can return ErrReplicating
-	// (wrapped in an os.PathError). This indicates that all data
-	// has been written, but the lease is still open for the file.
-	//
-	// It is safe in this case to either ignore the error (and let
-	// the lease expire on its own) or to call Close multiple
-	// times until it completes without an error. The Java client,
-	// for context, always chooses to retry, with exponential
-	// backoff.
-	err = o.fs.pacer.Call(func() (bool, error) {
-		err := out.Close()
-		if err == nil {
-			return false, nil
-		}
-		return errors.Is(err, hdfs.ErrReplicating), err
-	})
+	err = out.Close()
 	if err != nil {
 		cleanup()
 		return err

backend/hidrive/helpers.go

@@ -294,6 +294,15 @@ func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType Cop
 	return &result, nil
 }
 
+// copyDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) copyDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
 // moveDirectory moves the directory at the source-path to the destination-path and
 // returns the resulting api-object if successful.
 //

backend/hidrive/hidrive.go

@@ -2,7 +2,7 @@
 package hidrive
 
 // FIXME HiDrive only supports file or folder names of 255 characters or less.
-// Operations that create files or folders with longer names will throw an HTTP error:
+// Operations that create files oder folder with longer names will throw a HTTP error:
 // - 422 Unprocessable Entity
 // A more graceful way for rclone to handle this may be desirable.
 
@@ -338,7 +338,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("could not access root-prefix: %w", err)
 		}
 		if item.Type != api.HiDriveObjectTypeDirectory {
-			return nil, errors.New("the root-prefix needs to point to a valid directory or be empty")
+			return nil, errors.New("The root-prefix needs to point to a valid directory or be empty")
 		}
 	}
 

backend/http/http.go

@@ -36,7 +36,6 @@ func init() {
 		Name:        "http",
 		Description: "HTTP",
 		NewFs:       NewFs,
-		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "url",
 			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
@@ -211,42 +210,6 @@ func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Op
 	return createFileResult()
 }
 
-// Make the http connection with opt
-func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err error) {
-	if len(opt.Headers)%2 != 0 {
-		return false, errors.New("odd number of headers supplied")
-	}
-
-	if !strings.HasSuffix(opt.Endpoint, "/") {
-		opt.Endpoint += "/"
-	}
-
-	// Parse the endpoint and stick the root onto it
-	base, err := url.Parse(opt.Endpoint)
-	if err != nil {
-		return false, err
-	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
-	if err != nil {
-		return false, err
-	}
-
-	client := fshttp.NewClient(ctx)
-
-	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
-	fs.Debugf(nil, "Root: %s", endpoint)
-	u, err = url.Parse(endpoint)
-	if err != nil {
-		return false, err
-	}
-
-	// Update f with the new parameters
-	f.httpClient = client
-	f.endpoint = u
-	f.endpointURL = u.String()
-	return isFile, nil
-}
-
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -257,23 +220,47 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
+	if len(opt.Headers)%2 != 0 {
+		return nil, errors.New("odd number of headers supplied")
+	}
+
+	if !strings.HasSuffix(opt.Endpoint, "/") {
+		opt.Endpoint += "/"
+	}
+
+	// Parse the endpoint and stick the root onto it
+	base, err := url.Parse(opt.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	u, err := rest.URLJoin(base, rest.URLPathEscape(root))
+	if err != nil {
+		return nil, err
+	}
+
+	client := fshttp.NewClient(ctx)
+
+	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
+	fs.Debugf(nil, "Root: %s", endpoint)
+	u, err = url.Parse(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
 		name:        name,
 		root:        root,
 		opt:         *opt,
 		ci:          ci,
+		httpClient:  client,
+		endpoint:    u,
+		endpointURL: u.String(),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
 
-	// Make the http connection
-	isFile, err := f.httpConnection(ctx, opt)
-	if err != nil {
-		return nil, err
-	}
-
 	if isFile {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
@@ -508,7 +495,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					add(file)
 				case fs.ErrorNotAFile:
 					// ...found a directory not a file
-					add(fs.NewDir(remote, time.Time{}))
+					add(fs.NewDir(remote, timeUnset))
 				default:
 					fs.Debugf(remote, "skipping because of error: %v", err)
 				}
@@ -520,7 +507,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		name = strings.TrimRight(name, "/")
 		remote := path.Join(dir, name)
 		if isDir {
-			add(fs.NewDir(remote, time.Time{}))
+			add(fs.NewDir(remote, timeUnset))
 		} else {
 			in <- remote
 		}
@@ -698,66 +685,10 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return o.contentType
 }
 
-var commandHelp = []fs.CommandHelp{{
-	Name:  "set",
-	Short: "Set command for updating the config parameters.",
-	Long: `This set command can be used to update the config parameters
-for a running http backend.
-
-Usage Examples:
-
-    rclone backend set remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
-    rclone rc backend/command command=set fs=remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
-    rclone rc backend/command command=set fs=remote: -o url=https://example.com
-
-The option keys are named as they are in the config file.
-
-This rebuilds the connection to the http backend when it is called with
-the new parameters. Only new parameters need be passed as the values
-will default to those currently in use.
-
-It doesn't return anything.
-`,
-}}
-
-// Command the backend to run a named command
-//
-// The command run is name
-// args may be used to read arguments from
-// opts may be used to read optional arguments from
-//
-// The result should be capable of being JSON encoded
-// If it is a string or a []string it will be shown to the user
-// otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	switch name {
-	case "set":
-		newOpt := f.opt
-		err := configstruct.Set(configmap.Simple(opt), &newOpt)
-		if err != nil {
-			return nil, fmt.Errorf("reading config: %w", err)
-		}
-		_, err = f.httpConnection(ctx, &newOpt)
-		if err != nil {
-			return nil, fmt.Errorf("updating session: %w", err)
-		}
-		f.opt = newOpt
-		keys := []string{}
-		for k := range opt {
-			keys = append(keys, k)
-		}
-		fs.Logf(f, "Updated config values: %s", strings.Join(keys, ", "))
-		return nil, nil
-	default:
-		return nil, fs.ErrorCommandNotFound
-	}
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
-	_ fs.Commander   = &Fs{}
 )

backend/http/http_internal_test.go

@@ -33,9 +33,8 @@ var (
 	lineEndSize = 1
 )
 
-// prepareServer prepares the test server and shuts it down automatically
-// when the test completes.
-func prepareServer(t *testing.T) configmap.Simple {
+// prepareServer the test server and return a function to tidy it up afterwards
+func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	// file server for test/files
 	fileServer := http.FileServer(http.Dir(filesPath))
 
@@ -79,21 +78,20 @@ func prepareServer(t *testing.T) configmap.Simple {
 		"url":     ts.URL,
 		"headers": strings.Join(headers, ","),
 	}
-	t.Cleanup(ts.Close)
 
-	return m
+	// return a function to tidy up
+	return m, ts.Close
 }
 
-// prepare prepares the test server and shuts it down automatically
-// when the test completes.
-func prepare(t *testing.T) fs.Fs {
-	m := prepareServer(t)
+// prepare the test server and return a function to tidy it up afterwards
+func prepare(t *testing.T) (fs.Fs, func()) {
+	m, tidy := prepareServer(t)
 
 	// Instantiate it
 	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
-	return f
+	return f, tidy
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
@@ -136,19 +134,22 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 }
 
 func TestListRoot(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 	testListRoot(t, f, false)
 }
 
 func TestListRootNoSlash(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
 	f.(*Fs).opt.NoSlash = true
+	defer tidy()
 
 	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
@@ -165,7 +166,8 @@ func TestListSubDir(t *testing.T) {
 }
 
 func TestNewObject(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -192,7 +194,8 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	m := prepareServer(t)
+	m, tidy := prepareServer(t)
+	defer tidy()
 
 	for _, head := range []bool{false, true} {
 		if !head {
@@ -254,7 +257,8 @@ func TestOpen(t *testing.T) {
 }
 
 func TestMimeType(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -265,7 +269,8 @@ func TestMimeType(t *testing.T) {
 }
 
 func TestIsAFileRoot(t *testing.T) {
-	m := prepareServer(t)
+	m, tidy := prepareServer(t)
+	defer tidy()
 
 	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
@@ -274,7 +279,8 @@ func TestIsAFileRoot(t *testing.T) {
 }
 
 func TestIsAFileSubDir(t *testing.T) {
-	m := prepareServer(t)
+	m, tidy := prepareServer(t)
+	defer tidy()
 
 	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)

backend/imagekit/client/client.go

@@ -1,66 +0,0 @@
-// Package client provides a client for interacting with the ImageKit API.
-package client
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// ImageKit main struct
-type ImageKit struct {
-	Prefix        string
-	UploadPrefix  string
-	Timeout       int64
-	UploadTimeout int64
-	PrivateKey    string
-	PublicKey     string
-	URLEndpoint   string
-	HTTPClient    *rest.Client
-}
-
-// NewParams is a struct to define parameters to imagekit
-type NewParams struct {
-	PrivateKey  string
-	PublicKey   string
-	URLEndpoint string
-}
-
-// New returns ImageKit object from environment variables
-func New(ctx context.Context, params NewParams) (*ImageKit, error) {
-
-	privateKey := params.PrivateKey
-	publicKey := params.PublicKey
-	endpointURL := params.URLEndpoint
-
-	switch {
-	case privateKey == "":
-		return nil, fmt.Errorf("ImageKit.io URL endpoint is required")
-	case publicKey == "":
-		return nil, fmt.Errorf("ImageKit.io public key is required")
-	case endpointURL == "":
-		return nil, fmt.Errorf("ImageKit.io private key is required")
-	}
-
-	cliCtx, cliCfg := fs.AddConfig(ctx)
-
-	cliCfg.UserAgent = "rclone/imagekit"
-	client := rest.NewClient(fshttp.NewClient(cliCtx))
-
-	client.SetUserPass(privateKey, "")
-	client.SetHeader("Accept", "application/json")
-
-	return &ImageKit{
-		Prefix:        "https://api.imagekit.io/v2",
-		UploadPrefix:  "https://upload.imagekit.io/api/v2",
-		Timeout:       60,
-		UploadTimeout: 3600,
-		PrivateKey:    params.PrivateKey,
-		PublicKey:     params.PublicKey,
-		URLEndpoint:   params.URLEndpoint,
-		HTTPClient:    client,
-	}, nil
-}

backend/imagekit/client/media.go

@@ -1,252 +0,0 @@
-package client
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"time"
-
-	"github.com/rclone/rclone/lib/rest"
-	"gopkg.in/validator.v2"
-)
-
-// FilesOrFolderParam struct is a parameter type to ListFiles() function to search / list media library files.
-type FilesOrFolderParam struct {
-	Path        string `json:"path,omitempty"`
-	Limit       int    `json:"limit,omitempty"`
-	Skip        int    `json:"skip,omitempty"`
-	SearchQuery string `json:"searchQuery,omitempty"`
-}
-
-// AITag represents an AI tag for a media library file.
-type AITag struct {
-	Name       string  `json:"name"`
-	Confidence float32 `json:"confidence"`
-	Source     string  `json:"source"`
-}
-
-// File represents media library File details.
-type File struct {
-	FileID            string            `json:"fileId"`
-	Name              string            `json:"name"`
-	FilePath          string            `json:"filePath"`
-	Type              string            `json:"type"`
-	VersionInfo       map[string]string `json:"versionInfo"`
-	IsPrivateFile     *bool             `json:"isPrivateFile"`
-	CustomCoordinates *string           `json:"customCoordinates"`
-	URL               string            `json:"url"`
-	Thumbnail         string            `json:"thumbnail"`
-	FileType          string            `json:"fileType"`
-	Mime              string            `json:"mime"`
-	Height            int               `json:"height"`
-	Width             int               `json:"Width"`
-	Size              uint64            `json:"size"`
-	HasAlpha          bool              `json:"hasAlpha"`
-	CustomMetadata    map[string]any    `json:"customMetadata,omitempty"`
-	EmbeddedMetadata  map[string]any    `json:"embeddedMetadata"`
-	CreatedAt         time.Time         `json:"createdAt"`
-	UpdatedAt         time.Time         `json:"updatedAt"`
-	Tags              []string          `json:"tags"`
-	AITags            []AITag           `json:"AITags"`
-}
-
-// Folder represents media library Folder details.
-type Folder struct {
-	*File
-	FolderPath string `json:"folderPath"`
-}
-
-// CreateFolderParam represents parameter to create folder api
-type CreateFolderParam struct {
-	FolderName       string `validate:"nonzero" json:"folderName"`
-	ParentFolderPath string `validate:"nonzero" json:"parentFolderPath"`
-}
-
-// DeleteFolderParam represents parameter to delete folder api
-type DeleteFolderParam struct {
-	FolderPath string `validate:"nonzero" json:"folderPath"`
-}
-
-// MoveFolderParam represents parameter to move folder api
-type MoveFolderParam struct {
-	SourceFolderPath string `validate:"nonzero" json:"sourceFolderPath"`
-	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
-}
-
-// JobIDResponse respresents response struct with JobID for folder operations
-type JobIDResponse struct {
-	JobID string `json:"jobId"`
-}
-
-// JobStatus represents response Data to job status api
-type JobStatus struct {
-	JobID  string `json:"jobId"`
-	Type   string `json:"type"`
-	Status string `json:"status"`
-}
-
-// File represents media library File details.
-func (ik *ImageKit) File(ctx context.Context, fileID string) (*http.Response, *File, error) {
-	data := &File{}
-	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:       "GET",
-		Path:         fmt.Sprintf("/files/%s/details", fileID),
-		RootURL:      ik.Prefix,
-		IgnoreStatus: true,
-	}, nil, data)
-
-	return response, data, err
-}
-
-// Files retrieves media library files. Filter options can be supplied as FilesOrFolderParam.
-func (ik *ImageKit) Files(ctx context.Context, params FilesOrFolderParam, includeVersion bool) (*http.Response, *[]File, error) {
-	var SearchQuery = `type = "file"`
-
-	if includeVersion {
-		SearchQuery = `type IN ["file", "file-version"]`
-	}
-	if params.SearchQuery != "" {
-		SearchQuery = params.SearchQuery
-	}
-
-	parameters := url.Values{}
-
-	parameters.Set("skip", fmt.Sprintf("%d", params.Skip))
-	parameters.Set("limit", fmt.Sprintf("%d", params.Limit))
-	parameters.Set("path", params.Path)
-	parameters.Set("searchQuery", SearchQuery)
-
-	data := &[]File{}
-
-	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:     "GET",
-		Path:       "/files",
-		RootURL:    ik.Prefix,
-		Parameters: parameters,
-	}, nil, data)
-
-	return response, data, err
-}
-
-// DeleteFile removes file by FileID from media library
-func (ik *ImageKit) DeleteFile(ctx context.Context, fileID string) (*http.Response, error) {
-	var err error
-
-	if fileID == "" {
-		return nil, errors.New("fileID can not be empty")
-	}
-
-	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:     "DELETE",
-		Path:       fmt.Sprintf("/files/%s", fileID),
-		RootURL:    ik.Prefix,
-		NoResponse: true,
-	}, nil, nil)
-
-	return response, err
-}
-
-// Folders retrieves media library files. Filter options can be supplied as FilesOrFolderParam.
-func (ik *ImageKit) Folders(ctx context.Context, params FilesOrFolderParam) (*http.Response, *[]Folder, error) {
-	var SearchQuery = `type = "folder"`
-
-	if params.SearchQuery != "" {
-		SearchQuery = params.SearchQuery
-	}
-
-	parameters := url.Values{}
-
-	parameters.Set("skip", fmt.Sprintf("%d", params.Skip))
-	parameters.Set("limit", fmt.Sprintf("%d", params.Limit))
-	parameters.Set("path", params.Path)
-	parameters.Set("searchQuery", SearchQuery)
-
-	data := &[]Folder{}
-
-	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:     "GET",
-		Path:       "/files",
-		RootURL:    ik.Prefix,
-		Parameters: parameters,
-	}, nil, data)
-
-	if err != nil {
-		return resp, data, err
-	}
-
-	return resp, data, err
-}
-
-// CreateFolder creates a new folder in media library
-func (ik *ImageKit) CreateFolder(ctx context.Context, param CreateFolderParam) (*http.Response, error) {
-	var err error
-
-	if err = validator.Validate(&param); err != nil {
-		return nil, err
-	}
-
-	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:     "POST",
-		Path:       "/folder",
-		RootURL:    ik.Prefix,
-		NoResponse: true,
-	}, param, nil)
-
-	return response, err
-}
-
-// DeleteFolder removes the folder from media library
-func (ik *ImageKit) DeleteFolder(ctx context.Context, param DeleteFolderParam) (*http.Response, error) {
-	var err error
-
-	if err = validator.Validate(&param); err != nil {
-		return nil, err
-	}
-
-	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:     "DELETE",
-		Path:       "/folder",
-		RootURL:    ik.Prefix,
-		NoResponse: true,
-	}, param, nil)
-
-	return response, err
-}
-
-// MoveFolder moves given folder to new path in media library
-func (ik *ImageKit) MoveFolder(ctx context.Context, param MoveFolderParam) (*http.Response, *JobIDResponse, error) {
-	var err error
-	var response = &JobIDResponse{}
-
-	if err = validator.Validate(&param); err != nil {
-		return nil, nil, err
-	}
-
-	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:  "PUT",
-		Path:    "bulkJobs/moveFolder",
-		RootURL: ik.Prefix,
-	}, param, response)
-
-	return resp, response, err
-}
-
-// BulkJobStatus retrieves the status of a bulk job by job ID.
-func (ik *ImageKit) BulkJobStatus(ctx context.Context, jobID string) (*http.Response, *JobStatus, error) {
-	var err error
-	var response = &JobStatus{}
-
-	if jobID == "" {
-		return nil, nil, errors.New("jobId can not be blank")
-	}
-
-	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
-		Method:  "GET",
-		Path:    "bulkJobs/" + jobID,
-		RootURL: ik.Prefix,
-	}, nil, response)
-
-	return resp, response, err
-}

backend/imagekit/client/upload.go

@@ -1,96 +0,0 @@
-package client
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// UploadParam defines upload parameters
-type UploadParam struct {
-	FileName      string `json:"fileName"`
-	Folder        string `json:"folder,omitempty"` // default value:  /
-	Tags          string `json:"tags,omitempty"`
-	IsPrivateFile *bool  `json:"isPrivateFile,omitempty"` // default: false
-}
-
-// UploadResult defines the response structure for the upload API
-type UploadResult struct {
-	FileID       string            `json:"fileId"`
-	Name         string            `json:"name"`
-	URL          string            `json:"url"`
-	ThumbnailURL string            `json:"thumbnailUrl"`
-	Height       int               `json:"height"`
-	Width        int               `json:"Width"`
-	Size         uint64            `json:"size"`
-	FilePath     string            `json:"filePath"`
-	AITags       []map[string]any  `json:"AITags"`
-	VersionInfo  map[string]string `json:"versionInfo"`
-}
-
-// Upload uploads an asset to a imagekit account.
-//
-// The asset can be:
-//   - the actual data (io.Reader)
-//   - the Data URI (Base64 encoded), max ~60 MB (62,910,000 chars)
-//   - the remote FTP, HTTP or HTTPS URL address of an existing file
-//
-// https://docs.imagekit.io/api-reference/upload-file-api/server-side-file-upload
-func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadParam) (*http.Response, *UploadResult, error) {
-	var err error
-
-	if param.FileName == "" {
-		return nil, nil, errors.New("Upload: Filename is required")
-	}
-
-	// Initialize URL values
-	formParams := url.Values{}
-
-	formParams.Add("useUniqueFileName", fmt.Sprint(false))
-
-	// Add individual fields to URL values
-	if param.FileName != "" {
-		formParams.Add("fileName", param.FileName)
-	}
-
-	if param.Tags != "" {
-		formParams.Add("tags", param.Tags)
-	}
-
-	if param.Folder != "" {
-		formParams.Add("folder", param.Folder)
-	}
-
-	if param.IsPrivateFile != nil {
-		formParams.Add("isPrivateFile", fmt.Sprintf("%v", *param.IsPrivateFile))
-	}
-
-	response := &UploadResult{}
-
-	formReader, contentType, _, err := rest.MultipartUpload(ctx, file, formParams, "file", param.FileName)
-
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
-	}
-
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        "/files/upload",
-		RootURL:     ik.UploadPrefix,
-		Body:        formReader,
-		ContentType: contentType,
-	}
-
-	resp, err := ik.HTTPClient.CallJSON(ctx, &opts, nil, response)
-
-	if err != nil {
-		return resp, response, err
-	}
-
-	return resp, response, err
-}

backend/imagekit/client/url.go

@@ -1,72 +0,0 @@
-package client
-
-import (
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
-	neturl "net/url"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// URLParam represents parameters for generating url
-type URLParam struct {
-	Path            string
-	Src             string
-	URLEndpoint     string
-	Signed          bool
-	ExpireSeconds   int64
-	QueryParameters map[string]string
-}
-
-// URL generates url from URLParam
-func (ik *ImageKit) URL(params URLParam) (string, error) {
-	var resultURL string
-	var url *neturl.URL
-	var err error
-	var endpoint = params.URLEndpoint
-
-	if endpoint == "" {
-		endpoint = ik.URLEndpoint
-	}
-
-	endpoint = strings.TrimRight(endpoint, "/") + "/"
-
-	if params.QueryParameters == nil {
-		params.QueryParameters = make(map[string]string)
-	}
-
-	if url, err = neturl.Parse(params.Src); err != nil {
-		return "", err
-	}
-
-	query := url.Query()
-
-	for k, v := range params.QueryParameters {
-		query.Set(k, v)
-	}
-	url.RawQuery = query.Encode()
-	resultURL = url.String()
-
-	if params.Signed {
-		now := time.Now().Unix()
-
-		var expires = strconv.FormatInt(now+params.ExpireSeconds, 10)
-		var path = strings.Replace(resultURL, endpoint, "", 1)
-
-		path = path + expires
-		mac := hmac.New(sha1.New, []byte(ik.PrivateKey))
-		mac.Write([]byte(path))
-		signature := hex.EncodeToString(mac.Sum(nil))
-
-		if strings.Contains(resultURL, "?") {
-			resultURL = resultURL + "&" + fmt.Sprintf("ik-t=%s&ik-s=%s", expires, signature)
-		} else {
-			resultURL = resultURL + "?" + fmt.Sprintf("ik-t=%s&ik-s=%s", expires, signature)
-		}
-	}
-
-	return resultURL, nil
-}

backend/imagekit/imagekit.go

@@ -1,828 +0,0 @@
-// Package imagekit provides an interface to the ImageKit.io media library.
-package imagekit
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/backend/imagekit/client"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/readers"
-	"github.com/rclone/rclone/lib/version"
-)
-
-const (
-	minSleep      = 1 * time.Millisecond
-	maxSleep      = 100 * time.Millisecond
-	decayConstant = 2
-)
-
-var systemMetadataInfo = map[string]fs.MetadataHelp{
-	"btime": {
-		Help:     "Time of file birth (creation) read from Last-Modified header",
-		Type:     "RFC 3339",
-		Example:  "2006-01-02T15:04:05.999999999Z07:00",
-		ReadOnly: true,
-	},
-	"size": {
-		Help:     "Size of the object in bytes",
-		Type:     "int64",
-		ReadOnly: true,
-	},
-	"file-type": {
-		Help:     "Type of the file",
-		Type:     "string",
-		Example:  "image",
-		ReadOnly: true,
-	},
-	"height": {
-		Help:     "Height of the image or video in pixels",
-		Type:     "int",
-		ReadOnly: true,
-	},
-	"width": {
-		Help:     "Width of the image or video in pixels",
-		Type:     "int",
-		ReadOnly: true,
-	},
-	"has-alpha": {
-		Help:     "Whether the image has alpha channel or not",
-		Type:     "bool",
-		ReadOnly: true,
-	},
-	"tags": {
-		Help:     "Tags associated with the file",
-		Type:     "string",
-		Example:  "tag1,tag2",
-		ReadOnly: true,
-	},
-	"google-tags": {
-		Help:     "AI generated tags by Google Cloud Vision associated with the image",
-		Type:     "string",
-		Example:  "tag1,tag2",
-		ReadOnly: true,
-	},
-	"aws-tags": {
-		Help:     "AI generated tags by AWS Rekognition associated with the image",
-		Type:     "string",
-		Example:  "tag1,tag2",
-		ReadOnly: true,
-	},
-	"is-private-file": {
-		Help:     "Whether the file is private or not",
-		Type:     "bool",
-		ReadOnly: true,
-	},
-	"custom-coordinates": {
-		Help:     "Custom coordinates of the file",
-		Type:     "string",
-		Example:  "0,0,100,100",
-		ReadOnly: true,
-	},
-}
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "imagekit",
-		Description: "ImageKit.io",
-		NewFs:       NewFs,
-		MetadataInfo: &fs.MetadataInfo{
-			System: systemMetadataInfo,
-			Help:   `Any metadata supported by the underlying remote is read and written.`,
-		},
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "You can find your ImageKit.io URL endpoint in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
-				Required: true,
-			},
-			{
-				Name:      "public_key",
-				Help:      "You can find your ImageKit.io public key in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "private_key",
-				Help:      "You can find your ImageKit.io private key in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:     "only_signed",
-				Help:     "If you have configured `Restrict unsigned image URLs` in your dashboard settings, set this to true.",
-				Default:  false,
-				Advanced: true,
-			},
-			{
-				Name:     "versions",
-				Help:     "Include old versions in directory listings.",
-				Default:  false,
-				Advanced: true,
-			},
-			{
-				Name:     "upload_tags",
-				Help:     "Tags to add to the uploaded files, e.g. \"tag1,tag2\".",
-				Default:  "",
-				Advanced: true,
-			},
-			{
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.EncodeZero |
-					encoder.EncodeSlash |
-					encoder.EncodeQuestion |
-					encoder.EncodeHashPercent |
-					encoder.EncodeCtl |
-					encoder.EncodeDel |
-					encoder.EncodeDot |
-					encoder.EncodeDoubleQuote |
-					encoder.EncodePercent |
-					encoder.EncodeBackSlash |
-					encoder.EncodeDollar |
-					encoder.EncodeLtGt |
-					encoder.EncodeSquareBracket |
-					encoder.EncodeInvalidUtf8),
-			},
-		},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	Endpoint   string               `config:"endpoint"`
-	PublicKey  string               `config:"public_key"`
-	PrivateKey string               `config:"private_key"`
-	OnlySigned bool                 `config:"only_signed"`
-	Versions   bool                 `config:"versions"`
-	Enc        encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs represents a remote to ImageKit
-type Fs struct {
-	name     string           // name of remote
-	root     string           // root path
-	opt      Options          // parsed options
-	features *fs.Features     // optional features
-	ik       *client.ImageKit // ImageKit client
-	pacer    *fs.Pacer        // pacer for API calls
-}
-
-// Object describes a ImageKit file
-type Object struct {
-	fs          *Fs         // The Fs this object is part of
-	remote      string      // The remote path
-	filePath    string      // The path to the file
-	contentType string      // The content type of the object if known - may be ""
-	timestamp   time.Time   // The timestamp of the object if known - may be zero
-	file        client.File // The media file if known - may be nil
-	versionID   string      // If present this points to an object version
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-
-	if err != nil {
-		return nil, err
-	}
-
-	ik, err := client.New(ctx, client.NewParams{
-		URLEndpoint: opt.Endpoint,
-		PublicKey:   opt.PublicKey,
-		PrivateKey:  opt.PrivateKey,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	f := &Fs{
-		name:  name,
-		opt:   *opt,
-		ik:    ik,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-	}
-
-	f.root = path.Join("/", root)
-
-	f.features = (&fs.Features{
-		CaseInsensitive:         false,
-		DuplicateFiles:          false,
-		ReadMimeType:            true,
-		WriteMimeType:           false,
-		CanHaveEmptyDirectories: true,
-		BucketBased:             false,
-		ServerSideAcrossConfigs: false,
-		IsLocal:                 false,
-		SlowHash:                true,
-		ReadMetadata:            true,
-		WriteMetadata:           false,
-		UserMetadata:            false,
-		FilterAware:             true,
-		PartialUploads:          false,
-		NoMultiThreading:        false,
-	}).Fill(ctx, f)
-
-	if f.root != "/" {
-
-		r := f.root
-
-		folderPath := f.EncodePath(r[:strings.LastIndex(r, "/")+1])
-		fileName := f.EncodeFileName(r[strings.LastIndex(r, "/")+1:])
-
-		file := f.getFileByName(ctx, folderPath, fileName)
-
-		if file != nil {
-			newRoot := path.Dir(f.root)
-			f.root = newRoot
-			return f, fs.ErrorIsFile
-		}
-
-	}
-	return f, nil
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return strings.TrimLeft(f.root, "/")
-}
-
-// String returns a description of the FS
-func (f *Fs) String() string {
-	return fmt.Sprintf("FS imagekit: %s", f.root)
-}
-
-// Precision of the ModTimes in this Fs
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash types of the filesystem.
-func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet()
-}
-
-// Features returns the optional features of this Fs.
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-
-	remote := path.Join(f.root, dir)
-
-	remote = f.EncodePath(remote)
-
-	if remote != "/" {
-		parentFolderPath, folderName := path.Split(remote)
-		folderExists, err := f.getFolderByName(ctx, parentFolderPath, folderName)
-
-		if err != nil {
-			return make(fs.DirEntries, 0), err
-		}
-
-		if folderExists == nil {
-			return make(fs.DirEntries, 0), fs.ErrorDirNotFound
-		}
-	}
-
-	folders, folderError := f.getFolders(ctx, remote)
-
-	if folderError != nil {
-		return make(fs.DirEntries, 0), folderError
-	}
-
-	files, fileError := f.getFiles(ctx, remote, f.opt.Versions)
-
-	if fileError != nil {
-		return make(fs.DirEntries, 0), fileError
-	}
-
-	res := make([]fs.DirEntry, 0, len(folders)+len(files))
-
-	for _, folder := range folders {
-		folderPath := f.DecodePath(strings.TrimLeft(strings.Replace(folder.FolderPath, f.EncodePath(f.root), "", 1), "/"))
-		res = append(res, fs.NewDir(folderPath, folder.UpdatedAt))
-	}
-
-	for _, file := range files {
-		res = append(res, f.newObject(ctx, remote, file))
-	}
-
-	return res, nil
-}
-
-func (f *Fs) newObject(ctx context.Context, remote string, file client.File) *Object {
-	remoteFile := strings.TrimLeft(strings.Replace(file.FilePath, f.EncodePath(f.root), "", 1), "/")
-
-	folderPath, fileName := path.Split(remoteFile)
-
-	folderPath = f.DecodePath(folderPath)
-	fileName = f.DecodeFileName(fileName)
-
-	remoteFile = path.Join(folderPath, fileName)
-
-	if file.Type == "file-version" {
-		remoteFile = version.Add(remoteFile, file.UpdatedAt)
-
-		return &Object{
-			fs:          f,
-			remote:      remoteFile,
-			filePath:    file.FilePath,
-			contentType: file.Mime,
-			timestamp:   file.UpdatedAt,
-			file:        file,
-			versionID:   file.VersionInfo["id"],
-		}
-	}
-
-	return &Object{
-		fs:          f,
-		remote:      remoteFile,
-		filePath:    file.FilePath,
-		contentType: file.Mime,
-		timestamp:   file.UpdatedAt,
-		file:        file,
-	}
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error ErrorObjectNotFound.
-//
-// If remote points to a directory then it should return
-// ErrorIsDir if possible without doing any extra work,
-// otherwise ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	r := path.Join(f.root, remote)
-
-	folderPath, fileName := path.Split(r)
-
-	folderPath = f.EncodePath(folderPath)
-	fileName = f.EncodeFileName(fileName)
-
-	isFolder, err := f.getFolderByName(ctx, folderPath, fileName)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if isFolder != nil {
-		return nil, fs.ErrorIsDir
-	}
-
-	file := f.getFileByName(ctx, folderPath, fileName)
-
-	if file == nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-
-	return f.newObject(ctx, r, *file), nil
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
-// return an error or upload it properly (rather than e.g. calling panic).
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return uploadFile(ctx, f, in, src.Remote(), options...)
-}
-
-// Mkdir makes the directory (container, bucket)
-//
-// Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
-	remote := path.Join(f.root, dir)
-	parentFolderPath, folderName := path.Split(remote)
-
-	parentFolderPath = f.EncodePath(parentFolderPath)
-	folderName = f.EncodeFileName(folderName)
-
-	err = f.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		res, err = f.ik.CreateFolder(ctx, client.CreateFolderParam{
-			ParentFolderPath: parentFolderPath,
-			FolderName:       folderName,
-		})
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	return err
-}
-
-// Rmdir removes the directory (container, bucket) if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
-
-	entries, err := f.List(ctx, dir)
-
-	if err != nil {
-		return err
-	}
-
-	if len(entries) > 0 {
-		return errors.New("directory is not empty")
-	}
-
-	err = f.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		res, err = f.ik.DeleteFolder(ctx, client.DeleteFolderParam{
-			FolderPath: f.EncodePath(path.Join(f.root, dir)),
-		})
-
-		if res.StatusCode == http.StatusNotFound {
-			return false, fs.ErrorDirNotFound
-		}
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	return err
-}
-
-// Purge deletes all the files and the container
-//
-// Optional interface: Only implement this if you have a way of
-// deleting all the files quicker than just running Remove() on the
-// result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
-
-	remote := path.Join(f.root, dir)
-
-	err = f.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		res, err = f.ik.DeleteFolder(ctx, client.DeleteFolderParam{
-			FolderPath: f.EncodePath(remote),
-		})
-
-		if res.StatusCode == http.StatusNotFound {
-			return false, fs.ErrorDirNotFound
-		}
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	return err
-}
-
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-
-	duration := time.Duration(math.Abs(float64(expire)))
-
-	expireSeconds := duration.Seconds()
-
-	fileRemote := path.Join(f.root, remote)
-
-	folderPath, fileName := path.Split(fileRemote)
-	folderPath = f.EncodePath(folderPath)
-	fileName = f.EncodeFileName(fileName)
-
-	file := f.getFileByName(ctx, folderPath, fileName)
-
-	if file == nil {
-		return "", fs.ErrorObjectNotFound
-	}
-
-	// Pacer not needed as this doesn't use the API
-	url, err := f.ik.URL(client.URLParam{
-		Src:           file.URL,
-		Signed:        *file.IsPrivateFile || f.opt.OnlySigned,
-		ExpireSeconds: int64(expireSeconds),
-		QueryParameters: map[string]string{
-			"updatedAt": file.UpdatedAt.String(),
-		},
-	})
-
-	if err != nil {
-		return "", err
-	}
-
-	return url, nil
-}
-
-// Fs returns read only access to the Fs that this object is part of
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns the selected checksum of the file
-// If no checksum is available it returns ""
-func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
-	return "", hash.ErrUnsupported
-}
-
-// Storable says whether this object can be stored
-func (o *Object) Storable() bool {
-	return true
-}
-
-// String returns a description of the Object
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.file.Name
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification date of the file
-// It should return a best guess if one isn't available
-func (o *Object) ModTime(context.Context) time.Time {
-	return o.file.UpdatedAt
-}
-
-// Size returns the size of the file
-func (o *Object) Size() int64 {
-	return int64(o.file.Size)
-}
-
-// MimeType returns the MIME type of the file
-func (o *Object) MimeType(context.Context) string {
-	return o.contentType
-}
-
-// Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	// Offset and Count for range download
-	var offset int64
-	var count int64
-
-	fs.FixRangeOption(options, -1)
-	partialContent := false
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(-1)
-			partialContent = true
-		case *fs.SeekOption:
-			offset = x.Offset
-			partialContent = true
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-
-	// Pacer not needed as this doesn't use the API
-	url, err := o.fs.ik.URL(client.URLParam{
-		Src:    o.file.URL,
-		Signed: *o.file.IsPrivateFile || o.fs.opt.OnlySigned,
-		QueryParameters: map[string]string{
-			"tr":        "orig-true",
-			"updatedAt": o.file.UpdatedAt.String(),
-		},
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	client := &http.Client{}
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+count-1))
-	resp, err := client.Do(req)
-
-	if err != nil {
-		return nil, err
-	}
-
-	end := resp.ContentLength
-
-	if partialContent && resp.StatusCode == http.StatusOK {
-		skip := offset
-
-		if offset < 0 {
-			skip = end + offset + 1
-		}
-
-		_, err = io.CopyN(io.Discard, resp.Body, skip)
-		if err != nil {
-			if resp != nil {
-				_ = resp.Body.Close()
-			}
-			return nil, err
-		}
-
-		return readers.NewLimitedReadCloser(resp.Body, end-skip), nil
-	}
-
-	return resp.Body, nil
-}
-
-// Update in to the object with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
-// return an error or update the object properly (rather than e.g. calling panic).
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-
-	srcRemote := o.Remote()
-
-	remote := path.Join(o.fs.root, srcRemote)
-	folderPath, fileName := path.Split(remote)
-
-	UseUniqueFileName := new(bool)
-	*UseUniqueFileName = false
-
-	var resp *client.UploadResult
-
-	err = o.fs.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
-			FileName:      fileName,
-			Folder:        folderPath,
-			IsPrivateFile: o.file.IsPrivateFile,
-		})
-
-		return o.fs.shouldRetry(ctx, res, err)
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fileID := resp.FileID
-
-	_, file, err := o.fs.ik.File(ctx, fileID)
-
-	if err != nil {
-		return err
-	}
-
-	o.file = *file
-
-	return nil
-}
-
-// Remove this object
-func (o *Object) Remove(ctx context.Context) (err error) {
-	err = o.fs.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		res, err = o.fs.ik.DeleteFile(ctx, o.file.FileID)
-
-		return o.fs.shouldRetry(ctx, res, err)
-	})
-
-	return err
-}
-
-// SetModTime sets the metadata on the object to set the modification date
-func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, options ...fs.OpenOption) (fs.Object, error) {
-	remote := path.Join(f.root, srcRemote)
-	folderPath, fileName := path.Split(remote)
-
-	folderPath = f.EncodePath(folderPath)
-	fileName = f.EncodeFileName(fileName)
-
-	UseUniqueFileName := new(bool)
-	*UseUniqueFileName = false
-
-	err := f.pacer.Call(func() (bool, error) {
-		var res *http.Response
-		var err error
-		res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
-			FileName:      fileName,
-			Folder:        folderPath,
-			IsPrivateFile: &f.opt.OnlySigned,
-		})
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return f.NewObject(ctx, srcRemote)
-}
-
-// Metadata returns the metadata for the object
-func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
-
-	metadata.Set("btime", o.file.CreatedAt.Format(time.RFC3339))
-	metadata.Set("size", strconv.FormatUint(o.file.Size, 10))
-	metadata.Set("file-type", o.file.FileType)
-	metadata.Set("height", strconv.Itoa(o.file.Height))
-	metadata.Set("width", strconv.Itoa(o.file.Width))
-	metadata.Set("has-alpha", strconv.FormatBool(o.file.HasAlpha))
-
-	for k, v := range o.file.EmbeddedMetadata {
-		metadata.Set(k, fmt.Sprint(v))
-	}
-
-	if o.file.Tags != nil {
-		metadata.Set("tags", strings.Join(o.file.Tags, ","))
-	}
-
-	if o.file.CustomCoordinates != nil {
-		metadata.Set("custom-coordinates", *o.file.CustomCoordinates)
-	}
-
-	if o.file.IsPrivateFile != nil {
-		metadata.Set("is-private-file", strconv.FormatBool(*o.file.IsPrivateFile))
-	}
-
-	if o.file.AITags != nil {
-		googleTags := []string{}
-		awsTags := []string{}
-
-		for _, tag := range o.file.AITags {
-			if tag.Source == "google-auto-tagging" {
-				googleTags = append(googleTags, tag.Name)
-			} else if tag.Source == "aws-auto-tagging" {
-				awsTags = append(awsTags, tag.Name)
-			}
-		}
-
-		if len(googleTags) > 0 {
-			metadata.Set("google-tags", strings.Join(googleTags, ","))
-		}
-
-		if len(awsTags) > 0 {
-			metadata.Set("aws-tags", strings.Join(awsTags, ","))
-		}
-	}
-
-	return metadata, nil
-}
-
-// Copy src to this remote using server-side move operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantMove
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		return nil, fs.ErrorCantMove
-	}
-
-	file, err := srcObj.Open(ctx)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return uploadFile(ctx, f, file, remote)
-}
-
-// Check the interfaces are satisfied.
-var (
-	_ fs.Fs           = &Fs{}
-	_ fs.Purger       = &Fs{}
-	_ fs.PublicLinker = &Fs{}
-	_ fs.Object       = &Object{}
-	_ fs.Copier       = &Fs{}
-)

backend/imagekit/imagekit_test.go

@@ -1,18 +0,0 @@
-package imagekit
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-func TestIntegration(t *testing.T) {
-	debug := true
-	fstest.Verbose = &debug
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      "TestImageKit:",
-		NilObject:       (*Object)(nil),
-		SkipFsCheckWrap: true,
-	})
-}

backend/imagekit/util.go

@@ -1,193 +0,0 @@
-package imagekit
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/rclone/rclone/backend/imagekit/client"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/pacer"
-)
-
-func (f *Fs) getFiles(ctx context.Context, path string, includeVersions bool) (files []client.File, err error) {
-
-	files = make([]client.File, 0)
-
-	var hasMore = true
-
-	for hasMore {
-		err = f.pacer.Call(func() (bool, error) {
-			var data *[]client.File
-			var res *http.Response
-			res, data, err = f.ik.Files(ctx, client.FilesOrFolderParam{
-				Skip:  len(files),
-				Limit: 100,
-				Path:  path,
-			}, includeVersions)
-
-			hasMore = !(len(*data) == 0 || len(*data) < 100)
-
-			if len(*data) > 0 {
-				files = append(files, *data...)
-			}
-
-			return f.shouldRetry(ctx, res, err)
-		})
-	}
-
-	if err != nil {
-		return make([]client.File, 0), err
-	}
-
-	return files, nil
-}
-
-func (f *Fs) getFolders(ctx context.Context, path string) (folders []client.Folder, err error) {
-
-	folders = make([]client.Folder, 0)
-
-	var hasMore = true
-
-	for hasMore {
-		err = f.pacer.Call(func() (bool, error) {
-			var data *[]client.Folder
-			var res *http.Response
-			res, data, err = f.ik.Folders(ctx, client.FilesOrFolderParam{
-				Skip:  len(folders),
-				Limit: 100,
-				Path:  path,
-			})
-
-			hasMore = !(len(*data) == 0 || len(*data) < 100)
-
-			if len(*data) > 0 {
-				folders = append(folders, *data...)
-			}
-
-			return f.shouldRetry(ctx, res, err)
-		})
-	}
-
-	if err != nil {
-		return make([]client.Folder, 0), err
-	}
-
-	return folders, nil
-}
-
-func (f *Fs) getFileByName(ctx context.Context, path string, name string) (file *client.File) {
-
-	err := f.pacer.Call(func() (bool, error) {
-		res, data, err := f.ik.Files(ctx, client.FilesOrFolderParam{
-			Limit:       1,
-			Path:        path,
-			SearchQuery: fmt.Sprintf(`type = "file" AND name = %s`, strconv.Quote(name)),
-		}, false)
-
-		if len(*data) == 0 {
-			file = nil
-		} else {
-			file = &(*data)[0]
-		}
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	if err != nil {
-		return nil
-	}
-
-	return file
-}
-
-func (f *Fs) getFolderByName(ctx context.Context, path string, name string) (folder *client.Folder, err error) {
-	err = f.pacer.Call(func() (bool, error) {
-		res, data, err := f.ik.Folders(ctx, client.FilesOrFolderParam{
-			Limit:       1,
-			Path:        path,
-			SearchQuery: fmt.Sprintf(`type = "folder" AND name = %s`, strconv.Quote(name)),
-		})
-
-		if len(*data) == 0 {
-			folder = nil
-		} else {
-			folder = &(*data)[0]
-		}
-
-		return f.shouldRetry(ctx, res, err)
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return folder, nil
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	401, // Unauthorized (e.g. "Token has expired")
-	408, // Request Timeout
-	429, // Rate exceeded.
-	500, // Get occasional 500 Internal Server Error
-	503, // Service Unavailable
-	504, // Gateway Time-out
-}
-
-func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
-	if resp == nil {
-		return false
-	}
-	for _, e := range retryErrorCodes {
-		if resp.StatusCode == e {
-			return true
-		}
-	}
-	return false
-}
-
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-
-	if resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 503) {
-		var retryAfter = 1
-		retryAfterString := resp.Header.Get("X-RateLimit-Reset")
-		if retryAfterString != "" {
-			var err error
-			retryAfter, err = strconv.Atoi(retryAfterString)
-			if err != nil {
-				fs.Errorf(f, "Malformed %s header %q: %v", "X-RateLimit-Reset", retryAfterString, err)
-			}
-		}
-
-		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Millisecond)
-	}
-
-	return fserrors.ShouldRetry(err) || shouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-// EncodePath encapsulates the logic for encoding a path
-func (f *Fs) EncodePath(str string) string {
-	return f.opt.Enc.FromStandardPath(str)
-}
-
-// DecodePath encapsulates the logic for decoding a path
-func (f *Fs) DecodePath(str string) string {
-	return f.opt.Enc.ToStandardPath(str)
-}
-
-// EncodeFileName encapsulates the logic for encoding a file name
-func (f *Fs) EncodeFileName(str string) string {
-	return f.opt.Enc.FromStandardName(str)
-}
-
-// DecodeFileName encapsulates the logic for decoding a file name
-func (f *Fs) DecodeFileName(str string) string {
-	return f.opt.Enc.ToStandardName(str)
-}

backend/internetarchive/internetarchive.go

@@ -135,11 +135,9 @@ Owner is able to add custom keys. Metadata feature grabs all the keys including
 		Options: []fs.Option{{
 			Name: "access_key_id",
 			Help: "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
-			Sensitive: true,
 		}, {
 			Name: "secret_access_key",
 			Help: "IAS3 Secret Key (password).\n\nLeave blank for anonymous access.",
-			Sensitive: true,
 		}, {
 			// their official client (https://github.com/jjjake/internetarchive) hardcodes following the two
 			Name:     "endpoint",
@@ -802,7 +800,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
 	var mdata fs.Metadata
-	mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
+	mdata, err = fs.GetMetadataOptions(ctx, src, options)
 	if err == nil && mdata != nil {
 		for mk, mv := range mdata {
 			mk = strings.ToLower(mk)

backend/jottacloud/api/types.go

@@ -395,7 +395,7 @@ type JottaFile struct {
 	State           string    `xml:"currentRevision>state"`
 	CreatedAt       JottaTime `xml:"currentRevision>created"`
 	ModifiedAt      JottaTime `xml:"currentRevision>modified"`
-	UpdatedAt       JottaTime `xml:"currentRevision>updated"`
+	Updated         JottaTime `xml:"currentRevision>updated"`
 	Size            int64     `xml:"currentRevision>size"`
 	MimeType        string    `xml:"currentRevision>mime"`
 	MD5             string    `xml:"currentRevision>md5"`

backend/jottacloud/jottacloud.go

@@ -67,21 +67,13 @@ const (
 	legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
 	legacyConfigVersion         = 0
 
-	teliaseCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
-	teliaseCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
-	teliaseCloudClientID = "desktop"
-
-	telianoCloudTokenURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/token"
-	telianoCloudAuthURL  = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/auth"
-	telianoCloudClientID = "desktop"
+	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
+	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
+	teliaCloudClientID = "desktop"
 
 	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
 	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
 	tele2CloudClientID = "desktop"
-
-	onlimeCloudTokenURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/token"
-	onlimeCloudAuthURL  = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/auth"
-	onlimeCloudClientID = "desktop"
 )
 
 // Register with Fs
@@ -92,34 +84,7 @@ func init() {
 		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config:      Config,
-		MetadataInfo: &fs.MetadataInfo{
-			Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
-			System: map[string]fs.MetadataHelp{
-				"btime": {
-					Help:    "Time of file birth (creation), read from rclone metadata",
-					Type:    "RFC 3339",
-					Example: "2006-01-02T15:04:05.999999999Z07:00",
-				},
-				"mtime": {
-					Help:    "Time of last modification, read from rclone metadata",
-					Type:    "RFC 3339",
-					Example: "2006-01-02T15:04:05.999999999Z07:00",
-				},
-				"utime": {
-					Help:     "Time of last upload, when current revision was created, generated by backend",
-					Type:     "RFC 3339",
-					Example:  "2006-01-02T15:04:05.999999999Z07:00",
-					ReadOnly: true,
-				},
-				"content-type": {
-					Help:     "MIME type, also known as media type",
-					Type:     "string",
-					Example:  "text/plain",
-					ReadOnly: true,
-				},
-			},
-		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
@@ -154,7 +119,7 @@ func init() {
 			Default: (encoder.Display |
 				encoder.EncodeWin | // :?"*<>|
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -169,17 +134,11 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 			Value: "legacy",
 			Help:  "Legacy authentication.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
 		}, {
-			Value: "telia_se",
-			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud (Sweden).",
-		}, {
-			Value: "telia_no",
-			Help:  "Telia Sky authentication.\nUse this if you are using Telia Sky (Norway).",
+			Value: "telia",
+			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud.",
 		}, {
 			Value: "tele2",
 			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
-		}, {
-			Value: "onlime",
-			Help:  "Onlime Cloud authentication.\nUse this if you are using Onlime Cloud.",
 		}})
 	case "auth_type_done":
 		// Jump to next state according to config chosen
@@ -272,32 +231,17 @@ machines.`)
 			return nil, fmt.Errorf("error while saving token: %w", err)
 		}
 		return fs.ConfigGoto("choose_device")
-	case "telia_se": // telia_se cloud config
+	case "telia": // telia cloud config
 		m.Set("configVersion", fmt.Sprint(configVersion))
-		m.Set(configClientID, teliaseCloudClientID)
-		m.Set(configTokenURL, teliaseCloudTokenURL)
+		m.Set(configClientID, teliaCloudClientID)
+		m.Set(configTokenURL, teliaCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
 			OAuth2Config: &oauth2.Config{
 				Endpoint: oauth2.Endpoint{
-					AuthURL:  teliaseCloudAuthURL,
-					TokenURL: teliaseCloudTokenURL,
+					AuthURL:  teliaCloudAuthURL,
+					TokenURL: teliaCloudTokenURL,
 				},
-				ClientID:    teliaseCloudClientID,
-				Scopes:      []string{"openid", "jotta-default", "offline_access"},
-				RedirectURL: oauthutil.RedirectLocalhostURL,
-			},
-		})
-	case "telia_no": // telia_no cloud config
-		m.Set("configVersion", fmt.Sprint(configVersion))
-		m.Set(configClientID, telianoCloudClientID)
-		m.Set(configTokenURL, telianoCloudTokenURL)
-		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
-					AuthURL:  telianoCloudAuthURL,
-					TokenURL: telianoCloudTokenURL,
-				},
-				ClientID:    telianoCloudClientID,
+				ClientID:    teliaCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
@@ -317,21 +261,6 @@ machines.`)
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
 		})
-	case "onlime": // onlime cloud config
-		m.Set("configVersion", fmt.Sprint(configVersion))
-		m.Set(configClientID, onlimeCloudClientID)
-		m.Set(configTokenURL, onlimeCloudTokenURL)
-		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
-					AuthURL:  onlimeCloudAuthURL,
-					TokenURL: onlimeCloudTokenURL,
-				},
-				ClientID:    onlimeCloudClientID,
-				Scopes:      []string{"openid", "jotta-default", "offline_access"},
-				RedirectURL: oauthutil.RedirectLocalhostURL,
-			},
-		})
 	case "choose_device":
 		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
 Choosing no, the default, will let you access the storage used for the archive
@@ -524,9 +453,7 @@ type Object struct {
 	remote      string
 	hasMetaData bool
 	size        int64
-	createTime  time.Time
 	modTime     time.Time
-	updateTime  time.Time
 	md5         string
 	mimeType    string
 }
@@ -1001,9 +928,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		CanHaveEmptyDirectories: true,
 		ReadMimeType:            true,
 		WriteMimeType:           false,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            false,
 	}).Fill(ctx, f)
 	f.jfsSrv.SetErrorHandler(errorHandler)
 	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
@@ -1190,7 +1114,6 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
 			remote:      filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
 			size:        f.Size,
 			md5:         f.Checksum,
-			createTime:  time.Time(f.Created),
 			modTime:     time.Time(f.Modified),
 		})
 	}
@@ -1420,7 +1343,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // is currently in trash, but can be made to match, it will be
 // restored. Returns ErrorObjectNotFound if upload will be necessary
 // to get a matching remote file.
-func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
+func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:       "POST",
 		Path:         f.filePath(file),
@@ -1430,10 +1353,11 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Ti
 
 	opts.Parameters.Set("cphash", "true")
 
+	fileDate := api.JottaTime(modTime).String()
 	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
 	opts.ExtraHeaders["JMd5"] = md5
-	opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
-	opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
+	opts.ExtraHeaders["JCreated"] = fileDate
+	opts.ExtraHeaders["JModified"] = fileDate
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -1496,7 +1420,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
 	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
 		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
-		info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
+		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
 	}
 
 	if err != nil {
@@ -1759,9 +1683,7 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.size = info.Size
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
-	o.createTime = time.Time(info.CreatedAt)
 	o.modTime = time.Time(info.ModifiedAt)
-	o.updateTime = time.Time(info.UpdatedAt)
 	return nil
 }
 
@@ -1806,7 +1728,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	// (note that if size/md5 does not match, the file content will
 	// also be modified if deduplication is possible, i.e. it is
 	// important to use correct/latest values)
-	_, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
+	_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
 	if err != nil {
 		if err == fs.ErrorObjectNotFound {
 			// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
@@ -1916,12 +1838,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err == nil {
 			// if the object exists delete it
 			err = o.remove(ctx, true)
-			if err != nil && err != fs.ErrorObjectNotFound {
-				// if delete failed then report that, unless it was because the file did not exist after all
+			if err != nil {
 				return fmt.Errorf("failed to remove old object: %w", err)
 			}
-		} else if err != fs.ErrorObjectNotFound {
+		}
 		// if the object does not exist we can just continue but if the error is something different we should report that
+		if err != fs.ErrorObjectNotFound {
 			return err
 		}
 	}
@@ -1943,37 +1865,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Wrap the accounting back onto the stream
 		in = wrap(in)
 	}
-	// Fetch metadata if --metadata is in use
-	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
-	if err != nil {
-		return fmt.Errorf("failed to read metadata from source object: %w", err)
-	}
-	var createdTime string
-	var modTime string
-	if meta != nil {
-		if v, ok := meta["btime"]; ok {
-			t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
-			if err != nil {
-				fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
-			} else {
-				createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
-			}
-		}
-		if v, ok := meta["mtime"]; ok {
-			t, err := time.Parse(time.RFC3339Nano, v)
-			if err != nil {
-				fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
-			} else {
-				modTime = api.Rfc3339Time(t).String()
-			}
-		}
-	}
-	if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
-		modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
-	}
-	if createdTime == "" { // if no Created time set same as Modified
-		createdTime = modTime
-	}
 
 	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
@@ -1983,12 +1874,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options:      options,
 		ExtraHeaders: make(map[string]string),
 	}
+	fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
 
 	// the allocate request
 	var request = api.AllocateFileRequest{
 		Bytes:    size,
-		Created:  createdTime,
-		Modified: modTime,
+		Created:  fileDate,
+		Modified: fileDate,
 		Md5:      md5String,
 		Path:     o.fs.allocatePathRaw(o.remote, true),
 	}
@@ -2003,10 +1895,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRUPT, we must upload it.
-	// Else, if the file state is COMPLETE, we don't need to upload it because
-	// the content is already there, possibly it was created with deduplication,
-	// and also any metadata changes are already performed by the allocate request.
+	// If the file state is INCOMPLETE and CORRUPT, try to upload a then
 	if response.State != "COMPLETED" {
 		// how much do we still have to upload?
 		remainingBytes := size - response.ResumePos
@@ -2030,20 +1919,24 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 
 		// send the remaining bytes
-		_, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
+		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
 		if err != nil {
 			return err
 		}
 
-		// Upload response contains main metadata properties (size, md5 and modTime)
-		// which could be set back to the object, but it does not contain the
-		// necessary information to set the createTime and updateTime properties,
-		// so must therefore perform a read instead.
-	}
-	// in any case we must update the object meta data
+		// finally update the meta data
+		o.hasMetaData = true
+		o.size = result.Bytes
+		o.md5 = result.Md5
+		o.modTime = time.Unix(result.Modified/1000, 0)
+	} else {
+		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
 		return o.readMetaData(ctx, true)
 	}
 
+	return nil
+}
+
 func (o *Object) remove(ctx context.Context, hard bool) error {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -2058,17 +1951,10 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 		opts.Parameters.Set("dl", "true")
 	}
 
-	err := o.fs.pacer.Call(func() (bool, error) {
+	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
 		return shouldRetry(ctx, resp, err)
 	})
-	if apiErr, ok := err.(*api.Error); ok {
-		// attempting to hard delete will fail if path does not exist, but standard delete will succeed
-		if apiErr.StatusCode == http.StatusNotFound {
-			return fs.ErrorObjectNotFound
-		}
-	}
-	return err
 }
 
 // Remove an object
@@ -2076,22 +1962,6 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.remove(ctx, o.fs.opt.HardDelete)
 }
 
-// Metadata returns metadata for an object
-//
-// It should return nil if there is no Metadata
-func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
-	err = o.readMetaData(ctx, false)
-	if err != nil {
-		fs.Logf(o, "Failed to read metadata: %v", err)
-		return nil, err
-	}
-	metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
-	metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
-	metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
-	metadata.Set("content-type", o.mimeType)
-	return metadata, nil
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs           = (*Fs)(nil)
@@ -2106,5 +1976,4 @@ var (
 	_ fs.CleanUpper   = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
 	_ fs.MimeTyper    = (*Object)(nil)
-	_ fs.Metadataer   = (*Object)(nil)
 )

backend/jottacloud/jottacloud_internal_test.go

@@ -1,17 +1,11 @@
 package jottacloud
 
 import (
-	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
 	"testing"
-	"time"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -46,48 +40,3 @@ func TestReadMD5(t *testing.T) {
 		})
 	}
 }
-
-func (f *Fs) InternalTestMetadata(t *testing.T) {
-	ctx := context.Background()
-	contents := random.String(1000)
-
-	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
-	utime := time.Now()
-	metadata := fs.Metadata{
-		"btime": "2009-05-06T04:05:06.499999999Z",
-		"mtime": "2010-06-07T08:09:07.599999999Z",
-		//"utime" - read-only
-		//"content-type" - read-only
-	}
-	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata)
-	defer func() {
-		assert.NoError(t, obj.Remove(ctx))
-	}()
-	o := obj.(*Object)
-	gotMetadata, err := o.Metadata(ctx)
-	require.NoError(t, err)
-	for k, v := range metadata {
-		got := gotMetadata[k]
-		switch k {
-		case "btime":
-			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
-		case "mtime":
-			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
-		case "utime":
-			gotUtime := fstest.Time(got)
-			dt := gotUtime.Sub(utime)
-			assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("utime more than 1 minute out want %v got %v delta %v", utime, gotUtime, dt))
-			assert.True(t, fstest.Time(v).Equal(fstest.Time(got)))
-		case "content-type":
-			assert.True(t, o.MimeType(ctx) == got)
-		default:
-			assert.Equal(t, v, got, k)
-		}
-	}
-}
-
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Metadata", f.InternalTestMetadata)
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/koofr/koofr.go

@@ -64,7 +64,6 @@ func init() {
 			Name:     "user",
 			Help:     "Your user name.",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:       "password",
 			Help:       "Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password).",
@@ -377,7 +376,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	for i, file := range files {
 		remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name))
 		if file.Type == "dir" {
-			entries[i] = fs.NewDir(remote, time.Time{})
+			entries[i] = fs.NewDir(remote, time.Unix(0, 0))
 		} else {
 			entries[i] = &Object{
 				fs:     f,

backend/linkbox/linkbox.go

@@ -1,897 +0,0 @@
-// Package linkbox provides an interface to the linkbox.to Cloud storage system.
-//
-// API docs: https://www.linkbox.to/api-docs
-package linkbox
-
-/*
-   Extras
-   - PublicLink - NO - sharing doesn't share the actual file, only a page with it on
-   - Move - YES - have Move and Rename file APIs so is possible
-   - MoveDir - NO - probably not possible - have Move but no Rename
-*/
-
-import (
-	"bytes"
-	"context"
-	"crypto/md5"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/dircache"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	maxEntitiesPerPage = 1024
-	minSleep           = 200 * time.Millisecond
-	maxSleep           = 2 * time.Second
-	pacerBurst         = 1
-	linkboxAPIURL      = "https://www.linkbox.to/api/open/"
-	rootID             = "0" // ID of root directory
-)
-
-func init() {
-	fsi := &fs.RegInfo{
-		Name:        "linkbox",
-		Description: "Linkbox",
-		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Name:      "token",
-			Help:      "Token from https://www.linkbox.to/admin/account",
-			Sensitive: true,
-			Required:  true,
-		}},
-	}
-	fs.Register(fsi)
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	Token string `config:"token"`
-}
-
-// Fs stores the interface to the remote Linkbox files
-type Fs struct {
-	name     string
-	root     string
-	opt      Options            // options for this backend
-	features *fs.Features       // optional features
-	ci       *fs.ConfigInfo     // global config
-	srv      *rest.Client       // the connection to the server
-	dirCache *dircache.DirCache // Map of directory path to directory id
-	pacer    *fs.Pacer
-}
-
-// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
-type Object struct {
-	fs          *Fs
-	remote      string
-	size        int64
-	modTime     time.Time
-	contentType string
-	fullURL     string
-	dirID       int64
-	itemID      string // and these IDs are for files
-	id          int64  // these IDs appear to apply to directories
-	isDir       bool
-}
-
-// NewFs creates a new Fs object from the name and root. It connects to
-// the host specified in the config file.
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	root = strings.Trim(root, "/")
-	// Parse config into Options struct
-
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	ci := fs.GetConfig(ctx)
-
-	f := &Fs{
-		name: name,
-		opt:  *opt,
-		root: root,
-		ci:   ci,
-		srv:  rest.NewClient(fshttp.NewClient(ctx)),
-
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),
-	}
-	f.dirCache = dircache.New(root, rootID, f)
-
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-		CaseInsensitive:         true,
-	}).Fill(ctx, f)
-
-	// Find the current root
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		// Assume it is a file
-		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
-		tempF.root = newRoot
-		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(ctx, false)
-		if err != nil {
-			// No root so return old f
-			return f, nil
-		}
-		_, err := tempF.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				// File doesn't exist so return old f
-				return f, nil
-			}
-			return nil, err
-		}
-		f.features.Fill(ctx, &tempF)
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/rclone/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-type entity struct {
-	Type   string `json:"type"`
-	Name   string `json:"name"`
-	URL    string `json:"url"`
-	Ctime  int64  `json:"ctime"`
-	Size   int64  `json:"size"`
-	ID     int64  `json:"id"`
-	Pid    int64  `json:"pid"`
-	ItemID string `json:"item_id"`
-}
-
-// Return true if the entity is a directory
-func (e *entity) isDir() bool {
-	return e.Type == "dir" || e.Type == "sdir"
-}
-
-type data struct {
-	Entities []entity `json:"list"`
-}
-type fileSearchRes struct {
-	response
-	SearchData data `json:"data"`
-}
-
-// Set an object info from an entity
-func (o *Object) set(e *entity) {
-	o.modTime = time.Unix(e.Ctime, 0)
-	o.contentType = e.Type
-	o.size = e.Size
-	o.fullURL = e.URL
-	o.isDir = e.isDir()
-	o.id = e.ID
-	o.itemID = e.ItemID
-	o.dirID = e.Pid
-}
-
-// Call linkbox with the query in opts and return result
-//
-// This will be checked for error and an error will be returned if Status != 1
-func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result interface{}) error {
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	responser := result.(responser)
-	if responser.IsError() {
-		return responser
-	}
-	return nil
-}
-
-// list the objects into the function supplied
-//
-// If directories is set it only sends directories
-// User function to process a File item from listAll
-//
-// Should return true to finish processing
-type listAllFn func(*entity) bool
-
-// Search is a bit fussy about which characters match
-//
-// If the name doesn't match this then do an dir list instead
-var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ .]+$`)
-
-// Lists the directory required calling the user function on each item found
-//
-// If the user fn ever returns true then it early exits with found = true
-//
-// If you set name then search ignores dirID. name is a substring
-// search also so name="dir" matches "sub dir" also. This filters it
-// down so it only returns items in dirID
-func (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllFn) (found bool, err error) {
-	var (
-		pageNumber       = 0
-		numberOfEntities = maxEntitiesPerPage
-	)
-	name = strings.TrimSpace(name) // search doesn't like spaces
-	if !searchOK.MatchString(name) {
-		// If name isn't good then do an unbounded search
-		name = ""
-	}
-OUTER:
-	for numberOfEntities == maxEntitiesPerPage {
-		pageNumber++
-		opts := &rest.Opts{
-			Method:  "GET",
-			RootURL: linkboxAPIURL,
-			Path:    "file_search",
-			Parameters: url.Values{
-				"token":    {f.opt.Token},
-				"name":     {name},
-				"pid":      {dirID},
-				"pageNo":   {itoa(pageNumber)},
-				"pageSize": {itoa64(maxEntitiesPerPage)},
-			},
-		}
-
-		var responseResult fileSearchRes
-		err = getUnmarshaledResponse(ctx, f, opts, &responseResult)
-		if err != nil {
-			return false, fmt.Errorf("getting files failed: %w", err)
-
-		}
-
-		numberOfEntities = len(responseResult.SearchData.Entities)
-
-		for _, entity := range responseResult.SearchData.Entities {
-			if itoa64(entity.Pid) != dirID {
-				// when name != "" this returns from all directories, so ignore not this one
-				continue
-			}
-			if fn(&entity) {
-				found = true
-				break OUTER
-			}
-		}
-		if pageNumber > 100000 {
-			return false, fmt.Errorf("too many results")
-		}
-	}
-	return found, nil
-}
-
-// Turn 64 bit int to string
-func itoa64(i int64) string {
-	return strconv.FormatInt(i, 10)
-}
-
-// Turn int to string
-func itoa(i int) string {
-	return itoa64(int64(i))
-}
-
-func splitDirAndName(remote string) (dir string, name string) {
-	lastSlashPosition := strings.LastIndex(remote, "/")
-	if lastSlashPosition == -1 {
-		dir = ""
-		name = remote
-	} else {
-		dir = remote[:lastSlashPosition]
-		name = remote[lastSlashPosition+1:]
-	}
-
-	// fs.Debugf(nil, "splitDirAndName remote = {%s}, dir = {%s}, name = {%s}", remote, dir, name)
-
-	return dir, name
-}
-
-// FindLeaf finds a directory of name leaf in the folder with ID directoryID
-func (f *Fs) FindLeaf(ctx context.Context, directoryID, leaf string) (directoryIDOut string, found bool, err error) {
-	// Find the leaf in directoryID
-	found, err = f.listAll(ctx, directoryID, leaf, func(entity *entity) bool {
-		if entity.isDir() && strings.EqualFold(entity.Name, leaf) {
-			directoryIDOut = itoa64(entity.ID)
-			return true
-		}
-		return false
-	})
-	return directoryIDOut, found, err
-}
-
-// Returned from "folder_create"
-type folderCreateRes struct {
-	response
-	Data struct {
-		DirID int64 `json:"dirId"`
-	} `json:"data"`
-}
-
-// CreateDir makes a directory with dirID as parent and name leaf
-func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) {
-	// fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf)
-	opts := &rest.Opts{
-		Method:  "GET",
-		RootURL: linkboxAPIURL,
-		Path:    "folder_create",
-		Parameters: url.Values{
-			"token":       {f.opt.Token},
-			"name":        {leaf},
-			"pid":         {dirID},
-			"isShare":     {"0"},
-			"canInvite":   {"1"},
-			"canShare":    {"1"},
-			"withBodyImg": {"1"},
-			"desc":        {""},
-		},
-	}
-
-	response := folderCreateRes{}
-	err = getUnmarshaledResponse(ctx, f, opts, &response)
-	if err != nil {
-		// response status 1501 means that directory already exists
-		if response.Status == 1501 {
-			return newID, fmt.Errorf("couldn't find already created directory: %w", fs.ErrorDirNotFound)
-		}
-		return newID, fmt.Errorf("CreateDir failed: %w", err)
-
-	}
-	if response.Data.DirID == 0 {
-		return newID, fmt.Errorf("API returned 0 for ID of newly created directory")
-	}
-	return itoa64(response.Data.DirID), nil
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	// fs.Debugf(f, "List method dir = {%s}", dir)
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return nil, err
-	}
-	_, err = f.listAll(ctx, directoryID, "", func(entity *entity) bool {
-		remote := path.Join(dir, entity.Name)
-		if entity.isDir() {
-			id := itoa64(entity.ID)
-			modTime := time.Unix(entity.Ctime, 0)
-			d := fs.NewDir(remote, modTime).SetID(id).SetParentID(itoa64(entity.Pid))
-			entries = append(entries, d)
-			// cache the directory ID for later lookups
-			f.dirCache.Put(remote, id)
-		} else {
-			o := &Object{
-				fs:     f,
-				remote: remote,
-			}
-			o.set(entity)
-			entries = append(entries, o)
-		}
-		return false
-	})
-	if err != nil {
-		return nil, err
-	}
-	return entries, nil
-}
-
-// get an entity with leaf from dirID
-func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string, token string) (*entity, error) {
-	var result *entity
-	var resultErr = fs.ErrorObjectNotFound
-	_, err := f.listAll(ctx, directoryID, leaf, func(entity *entity) bool {
-		if strings.EqualFold(entity.Name, leaf) {
-			// fs.Debugf(f, "getObject found entity.Name {%s} name {%s}", entity.Name, name)
-			if entity.isDir() {
-				result = nil
-				resultErr = fs.ErrorIsDir
-			} else {
-				result = entity
-				resultErr = nil
-			}
-			return true
-		}
-		return false
-	})
-	if err != nil {
-		return nil, err
-	}
-	return result, resultErr
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error ErrorObjectNotFound.
-//
-// If remote points to a directory then it should return
-// ErrorIsDir if possible without doing any extra work,
-// otherwise ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, dirID, err := f.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		if err == fs.ErrorDirNotFound {
-			return nil, fs.ErrorObjectNotFound
-		}
-		return nil, err
-	}
-
-	entity, err := getEntity(ctx, f, leaf, dirID, f.opt.Token)
-	if err != nil {
-		return nil, err
-	}
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-	o.set(entity)
-	return o, nil
-}
-
-// Mkdir makes the directory (container, bucket)
-//
-// Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	_, err := f.dirCache.FindDir(ctx, dir, true)
-	return err
-}
-
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
-	if check {
-		entries, err := f.List(ctx, dir)
-		if err != nil {
-			return err
-		}
-		if len(entries) != 0 {
-			return fs.ErrorDirectoryNotEmpty
-		}
-	}
-
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-	opts := &rest.Opts{
-		Method:  "GET",
-		RootURL: linkboxAPIURL,
-		Path:    "folder_del",
-		Parameters: url.Values{
-			"token":  {f.opt.Token},
-			"dirIds": {directoryID},
-		},
-	}
-
-	response := response{}
-	err = getUnmarshaledResponse(ctx, f, opts, &response)
-	if err != nil {
-		// Linkbox has some odd error returns here
-		if response.Status == 403 || response.Status == 500 {
-			return fs.ErrorDirNotFound
-		}
-		return fmt.Errorf("purge error: %w", err)
-	}
-
-	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Rmdir removes the directory (container, bucket) if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
-}
-
-// SetModTime sets modTime on a particular file
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-// Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	var res *http.Response
-	downloadURL := o.fullURL
-	if downloadURL == "" {
-		_, name := splitDirAndName(o.Remote())
-		newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID), o.fs.opt.Token)
-		if err != nil {
-			return nil, err
-		}
-		if newObject == nil {
-			// fs.Debugf(o.fs, "Open entity is empty: name = {%s}", name)
-			return nil, fs.ErrorObjectNotFound
-		}
-
-		downloadURL = newObject.URL
-	}
-
-	opts := &rest.Opts{
-		Method:  "GET",
-		RootURL: downloadURL,
-		Options: options,
-	}
-
-	err := o.fs.pacer.Call(func() (bool, error) {
-		var err error
-		res, err = o.fs.srv.Call(ctx, opts)
-		return o.fs.shouldRetry(ctx, res, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("Open failed: %w", err)
-	}
-
-	return res.Body, nil
-}
-
-// Update in to the object with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
-// return an error or update the object properly (rather than e.g. calling panic).
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	size := src.Size()
-	if size == 0 {
-		return fs.ErrorCantUploadEmptyFiles
-	} else if size < 0 {
-		return fmt.Errorf("can't upload files of unknown length")
-	}
-
-	remote := o.Remote()
-
-	// remove the file if it exists
-	if o.itemID != "" {
-		fs.Debugf(o, "Update: removing old file")
-		err = o.Remove(ctx)
-		if err != nil {
-			fs.Errorf(o, "Update: failed to remove existing file: %v", err)
-		}
-		o.itemID = ""
-	} else {
-		tmpObject, err := o.fs.NewObject(ctx, remote)
-		if err == nil {
-			fs.Debugf(o, "Update: removing old file")
-			err = tmpObject.Remove(ctx)
-			if err != nil {
-				fs.Errorf(o, "Update: failed to remove existing file: %v", err)
-			}
-		}
-	}
-
-	first10m := io.LimitReader(in, 10_485_760)
-	first10mBytes, err := io.ReadAll(first10m)
-	if err != nil {
-		return fmt.Errorf("Update err in reading file: %w", err)
-	}
-
-	// get upload authorization (step 1)
-	opts := &rest.Opts{
-		Method:  "GET",
-		RootURL: linkboxAPIURL,
-		Path:    "get_upload_url",
-		Options: options,
-		Parameters: url.Values{
-			"token":           {o.fs.opt.Token},
-			"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
-			"fileSize":        {itoa64(size)},
-		},
-	}
-
-	getFirstStepResult := getUploadURLResponse{}
-	err = getUnmarshaledResponse(ctx, o.fs, opts, &getFirstStepResult)
-	if err != nil {
-		if getFirstStepResult.Status != 600 {
-			return fmt.Errorf("Update err in unmarshaling response: %w", err)
-		}
-	}
-
-	switch getFirstStepResult.Status {
-	case 1:
-		// upload file using link from first step
-		var res *http.Response
-
-		file := io.MultiReader(bytes.NewReader(first10mBytes), in)
-
-		opts := &rest.Opts{
-			Method:        "PUT",
-			RootURL:       getFirstStepResult.Data.SignURL,
-			Options:       options,
-			Body:          file,
-			ContentLength: &size,
-		}
-
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			res, err = o.fs.srv.Call(ctx, opts)
-			return o.fs.shouldRetry(ctx, res, err)
-		})
-
-		if err != nil {
-			return fmt.Errorf("update err in uploading file: %w", err)
-		}
-
-		_, err = io.ReadAll(res.Body)
-		if err != nil {
-			return fmt.Errorf("update err in reading response: %w", err)
-		}
-
-	case 600:
-		// Status means that we don't need to upload file
-		// We need only to make second step
-	default:
-		return fmt.Errorf("got unexpected message from Linkbox: %s", getFirstStepResult.Message)
-	}
-
-	leaf, dirID, err := o.fs.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		return err
-	}
-
-	// create file item at Linkbox (second step)
-	opts = &rest.Opts{
-		Method:  "GET",
-		RootURL: linkboxAPIURL,
-		Path:    "folder_upload_file",
-		Options: options,
-		Parameters: url.Values{
-			"token":           {o.fs.opt.Token},
-			"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
-			"fileSize":        {itoa64(size)},
-			"pid":             {dirID},
-			"diyName":         {leaf},
-		},
-	}
-
-	getSecondStepResult := getUploadURLResponse{}
-	err = getUnmarshaledResponse(ctx, o.fs, opts, &getSecondStepResult)
-	if err != nil {
-		return fmt.Errorf("Update second step failed: %w", err)
-	}
-
-	// Try a few times to read the object after upload for eventual consistency
-	const maxTries = 10
-	var sleepTime = 100 * time.Millisecond
-	var entity *entity
-	for try := 1; try <= maxTries; try++ {
-		entity, err = getEntity(ctx, o.fs, leaf, dirID, o.fs.opt.Token)
-		if err == nil {
-			break
-		}
-		if err != fs.ErrorObjectNotFound {
-			return fmt.Errorf("Update failed to read object: %w", err)
-		}
-		fs.Debugf(o, "Trying to read object after upload: try again in %v (%d/%d)", sleepTime, try, maxTries)
-		time.Sleep(sleepTime)
-		sleepTime *= 2
-	}
-	if err != nil {
-		return err
-	}
-	o.set(entity)
-	return nil
-}
-
-// Remove this object
-func (o *Object) Remove(ctx context.Context) error {
-	opts := &rest.Opts{
-		Method:  "GET",
-		RootURL: linkboxAPIURL,
-		Path:    "file_del",
-		Parameters: url.Values{
-			"token":   {o.fs.opt.Token},
-			"itemIds": {o.itemID},
-		},
-	}
-	requestResult := getUploadURLResponse{}
-	err := getUnmarshaledResponse(ctx, o.fs, opts, &requestResult)
-	if err != nil {
-		return fmt.Errorf("could not Remove: %w", err)
-
-	}
-	return nil
-}
-
-// ModTime returns the modification time of the remote http file
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// Remote the name of the remote HTTP file, relative to the fs root
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// Size returns the size in bytes of the remote http file
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// String returns the URL to the remote HTTP file
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Fs is the filesystem this remote http file object is located within
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns "" since HTTP (in Go or OpenSSH) doesn't support remote calculation of hashes
-func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
-	return "", hash.ErrUnsupported
-}
-
-// Storable returns whether the remote http file is a regular file
-// (not a directory, symbolic link, block device, character device, named pipe, etc.)
-func (o *Object) Storable() bool {
-	return true
-}
-
-// Features returns the optional features of this Fs
-// Info provides a read only interface to information about a filesystem.
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Name of the remote (as passed into NewFs)
-// Name returns the configured name of the file system
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a description of the FS
-func (f *Fs) String() string {
-	return fmt.Sprintf("Linkbox root '%s'", f.root)
-}
-
-// Precision of the ModTimes in this Fs
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns hash.HashNone to indicate remote hashing is unavailable
-// Returns the supported hash types of the filesystem
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.None)
-}
-
-/*
-	{
-	  "data": {
-	    "signUrl": "http://xx -- Then CURL PUT your file with sign url "
-	  },
-	  "msg": "please use this url to upload (PUT method)",
-	  "status": 1
-	}
-*/
-
-// All messages have these items
-type response struct {
-	Message string `json:"msg"`
-	Status  int    `json:"status"`
-}
-
-// IsError returns whether response represents an error
-func (r *response) IsError() bool {
-	return r.Status != 1
-}
-
-// Error returns the error state of this response
-func (r *response) Error() string {
-	return fmt.Sprintf("Linkbox error %d: %s", r.Status, r.Message)
-}
-
-// responser is interface covering the response so we can use it when it is embedded.
-type responser interface {
-	IsError() bool
-	Error() string
-}
-
-type getUploadURLData struct {
-	SignURL string `json:"signUrl"`
-}
-
-type getUploadURLResponse struct {
-	response
-	Data getUploadURLData `json:"data"`
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
-// return an error or upload it properly (rather than e.g. calling panic).
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	o := &Object{
-		fs:     f,
-		remote: src.Remote(),
-		size:   src.Size(),
-	}
-	dir, _ := splitDirAndName(src.Remote())
-	err := f.Mkdir(ctx, dir)
-	if err != nil {
-		return nil, err
-	}
-	err = o.Update(ctx, in, src, options...)
-	return o, err
-}
-
-// Purge all files in the directory specified
-//
-// Implement this if you have a way of deleting all the files
-// quicker than just running Remove() on the result of List()
-//
-// Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry determines whether a given err rates being retried
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-// DirCacheFlush resets the directory cache - used in testing as an
-// optional interface
-func (f *Fs) DirCacheFlush() {
-	f.dirCache.ResetRoot()
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs              = &Fs{}
-	_ fs.Purger          = &Fs{}
-	_ fs.DirCacheFlusher = &Fs{}
-	_ fs.Object          = &Object{}
-)

backend/linkbox/linkbox_test.go

@@ -1,17 +0,0 @@
-// Test Linkbox filesystem interface
-package linkbox_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/linkbox"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestLinkbox:",
-		NilObject:  (*linkbox.Object)(nil),
-	})
-}

backend/local/local.go

@@ -13,7 +13,6 @@ import (
 	"runtime"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 	"unicode/utf8"
 
@@ -146,11 +145,6 @@ time we:
 - Only checksum the size that stat gave
 - Don't update the stat info for the file
 
-**NB** do not use this flag on a Windows Volume Shadow (VSS). For some
-unknown reason, files in a VSS sometimes show different sizes from the
-directory listing (where the initial stat value comes from on Windows)
-and when stat is called on them directly. Other copy tools always use
-the direct stat value and setting this flag will disable that.
 `,
 			Default:  false,
 			Advanced: true,
@@ -249,7 +243,7 @@ type Fs struct {
 	precision      time.Duration       // precision of local filesystem
 	warnedMu       sync.Mutex          // used for locking access to 'warned'.
 	warned         map[string]struct{} // whether we have warned about this string
-	xattrSupported atomic.Int32        // whether xattrs are supported
+	xattrSupported int32               // whether xattrs are supported (atomic access)
 
 	// do os.Lstat or os.Stat
 	lstat        func(name string) (os.FileInfo, error)
@@ -272,10 +266,7 @@ type Object struct {
 
 // ------------------------------------------------------------
 
-var (
-	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
-)
+var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -297,7 +288,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		lstat:  os.Lstat,
 	}
 	if xattrSupported {
-		f.xattrSupported.Store(1)
+		f.xattrSupported = 1
 	}
 	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
@@ -309,7 +300,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		WriteMetadata:           true,
 		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
 		FilterAware:             true,
-		PartialUploads:          true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -320,16 +310,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	// Check to see if this is a .rclonelink if not found
-	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
-	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
-		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
-	}
 	if err == nil && f.isRegular(fi.Mode()) {
-		// Handle the odd case, that a symlink was specified by name without the link suffix
-		if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-			return nil, errLinksNeedsSuffix
-		}
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -522,7 +503,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 								continue
 							}
 						}
-						fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
+						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
@@ -543,10 +524,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
-				// Quietly skip errors on excluded files and directories
-				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
-					continue
-				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
@@ -559,6 +536,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				}
 				mode = fi.Mode()
 			}
+			// Don't include non directory if not included
+			// we leave directory filtering to the layer above
+			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
+				continue
+			}
 			if fi.IsDir() {
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
@@ -571,11 +553,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
-				// Don't include non directory if not included
-				// we leave directory filtering to the layer above
-				if useFilter && !filter.IncludeRemote(newRemote) {
-					continue
-				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err
@@ -647,13 +624,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // If it isn't empty it will return an error
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	localPath := f.localPath(dir)
-	if fi, err := os.Stat(localPath); err != nil {
-		return err
-	} else if !fi.IsDir() {
-		return fs.ErrorIsFile
-	}
-	return os.Remove(localPath)
+	return os.Remove(f.localPath(dir))
 }
 
 // Precision of the file system
@@ -1128,12 +1099,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 	}
 
-	// Update the file info before we start reading
-	err = o.lstat()
-	if err != nil {
-		return nil, err
-	}
-
 	// If not checking updated then limit to current size.  This means if
 	// file is being extended, readers will read a o.Size() bytes rather
 	// than the new size making for a consistent upload.
@@ -1298,7 +1263,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	// Fetch and set metadata if --metadata is in use
-	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
+	meta, err := fs.GetMetadataOptions(ctx, src, options)
 	if err != nil {
 		return fmt.Errorf("failed to read metadata from source object: %w", err)
 	}

backend/local/local_internal_test.go

@@ -14,12 +14,10 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
@@ -35,6 +33,7 @@ func TestMain(m *testing.M) {
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
@@ -79,6 +78,7 @@ func TestUpdatingCheck(t *testing.T) {
 func TestSymlink(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	f := r.Flocal.(*Fs)
 	dir := f.root
 
@@ -147,20 +147,6 @@ func TestSymlink(t *testing.T) {
 	_, err = r.Flocal.NewObject(ctx, "symlink2.txt")
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
-	// Check that NewFs works with the suffixed version and --links
-	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
-		"links": "true",
-	})
-	require.Equal(t, fs.ErrorIsFile, err)
-	require.Equal(t, dir, f2.(*Fs).root)
-
-	// Check that NewFs doesn't see the non suffixed version with --links
-	f2, err = NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"), configmap.Simple{
-		"links": "true",
-	})
-	require.Equal(t, errLinksNeedsSuffix, err)
-	require.Nil(t, f2)
-
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
@@ -191,6 +177,7 @@ func TestSymlinkError(t *testing.T) {
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -221,6 +208,7 @@ func TestHashOnUpdate(t *testing.T) {
 func TestHashOnDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -249,6 +237,7 @@ func TestHashOnDelete(t *testing.T) {
 func TestMetadata(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "metafile.txt"
 	when := time.Now()
 	const dayLength = len("2001-01-01")
@@ -383,6 +372,7 @@ func TestMetadata(t *testing.T) {
 func TestFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	when := time.Now()
 	r.WriteFile("included", "included file", when)
 	r.WriteFile("excluded", "excluded file", when)
@@ -411,147 +401,3 @@ func TestFilter(t *testing.T) {
 	sort.Sort(entries)
 	require.Equal(t, "[included]", fmt.Sprint(entries))
 }
-
-func testFilterSymlink(t *testing.T, copyLinks bool) {
-	ctx := context.Background()
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	when := time.Now()
-	f := r.Flocal.(*Fs)
-
-	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
-	r.WriteFile("included.file", "included file", when)
-	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
-	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
-	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
-	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
-
-	defer func() {
-		// Reset -L/-l mode
-		f.opt.FollowSymlinks = false
-		f.opt.TranslateSymlinks = false
-		f.lstat = os.Lstat
-	}()
-	if copyLinks {
-		// Set fs into "-L" mode
-		f.opt.FollowSymlinks = true
-		f.opt.TranslateSymlinks = false
-		f.lstat = os.Stat
-	} else {
-		// Set fs into "-l" mode
-		f.opt.FollowSymlinks = false
-		f.opt.TranslateSymlinks = true
-		f.lstat = os.Lstat
-	}
-
-	// Check set up for filtering
-	assert.True(t, f.Features().FilterAware)
-
-	// Reset global error count
-	accounting.Stats(ctx).ResetErrors()
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	// Add a filter
-	ctx, fi := filter.AddConfig(ctx)
-	require.NoError(t, fi.AddRule("+ included.file"))
-	require.NoError(t, fi.AddRule("+ included.dir/**"))
-	if copyLinks {
-		require.NoError(t, fi.AddRule("+ included.file.link"))
-		require.NoError(t, fi.AddRule("+ included.dir.link/**"))
-	} else {
-		require.NoError(t, fi.AddRule("+ included.file.link.rclonelink"))
-		require.NoError(t, fi.AddRule("+ included.dir.link.rclonelink"))
-	}
-	require.NoError(t, fi.AddRule("- *"))
-
-	// Check listing without use filter flag
-	entries, err := f.List(ctx, "")
-	require.NoError(t, err)
-
-	if copyLinks {
-		// Check 1 global errors one for each dangling symlink
-		assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
-	} else {
-		// Check 0 global errors as dangling symlink copied properly
-		assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-	}
-	accounting.Stats(ctx).ResetErrors()
-
-	sort.Sort(entries)
-	if copyLinks {
-		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
-	} else {
-		require.Equal(t, "[dangling.link.rclonelink included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
-	}
-
-	// Add user filter flag
-	ctx = filter.SetUseFilter(ctx, true)
-
-	// Check listing with use filter flag
-	entries, err = f.List(ctx, "")
-	require.NoError(t, err)
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	sort.Sort(entries)
-	if copyLinks {
-		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
-	} else {
-		require.Equal(t, "[included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
-	}
-
-	// Check listing through a symlink still works
-	entries, err = f.List(ctx, "included.dir")
-	require.NoError(t, err)
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	sort.Sort(entries)
-	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
-}
-
-func TestFilterSymlinkCopyLinks(t *testing.T) {
-	testFilterSymlink(t, true)
-}
-
-func TestFilterSymlinkLinks(t *testing.T) {
-	testFilterSymlink(t, false)
-}
-
-func TestCopySymlink(t *testing.T) {
-	ctx := context.Background()
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	when := time.Now()
-	f := r.Flocal.(*Fs)
-
-	// Create a file and a symlink to it
-	r.WriteFile("src/file.txt", "hello world", when)
-	require.NoError(t, os.Symlink("file.txt", filepath.Join(r.LocalName, "src", "link.txt")))
-	defer func() {
-		// Reset -L/-l mode
-		f.opt.FollowSymlinks = false
-		f.opt.TranslateSymlinks = false
-		f.lstat = os.Lstat
-	}()
-
-	// Set fs into "-l/--links" mode
-	f.opt.FollowSymlinks = false
-	f.opt.TranslateSymlinks = true
-	f.lstat = os.Lstat
-
-	// Create dst
-	require.NoError(t, f.Mkdir(ctx, "dst"))
-
-	// Do copy from src into dst
-	src, err := f.NewObject(ctx, "src/link.txt.rclonelink")
-	require.NoError(t, err)
-	require.NotNil(t, src)
-	dst, err := operations.Copy(ctx, f, nil, "dst/link.txt.rclonelink", src)
-	require.NoError(t, err)
-	require.NotNil(t, dst)
-
-	// Test that we made a symlink and it has the right contents
-	dstPath := filepath.Join(r.LocalName, "dst", "link.txt")
-	linkContents, err := os.Readlink(dstPath)
-	require.NoError(t, err)
-	assert.Equal(t, "file.txt", linkContents)
-}

backend/local/metadata_linux.go

@@ -5,7 +5,6 @@ package local
 
 import (
 	"fmt"
-	"runtime"
 	"sync"
 	"time"
 
@@ -24,7 +23,7 @@ func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
 		// Check statx() is available as it was only introduced in kernel 4.11
 		// If not, fall back to fstatat() which was introduced in 2.6.16 which is guaranteed for all Go versions
 		var stat unix.Statx_t
-		if runtime.GOOS != "android" && unix.Statx(unix.AT_FDCWD, ".", 0, unix.STATX_ALL, &stat) != unix.ENOSYS {
+		if unix.Statx(unix.AT_FDCWD, ".", 0, unix.STATX_ALL, &stat) != unix.ENOSYS {
 			readMetadataFromFileFn = readMetadataFromFileStatx
 		} else {
 			readMetadataFromFileFn = readMetadataFromFileFstatat
@@ -92,7 +91,7 @@ func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
 		// The types of t.Sec and t.Nsec vary from int32 to int64 on
 		// different Linux architectures so we need to cast them to
 		// int64 here and hence need to quiet the linter about
-		// unnecessary casts.
+		// unecessary casts.
 		//
 		// nolint: unconvert
 		m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat))

backend/local/xattr.go

@@ -6,6 +6,7 @@ package local
 import (
 	"fmt"
 	"strings"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/pkg/xattr"
@@ -27,7 +28,7 @@ func (f *Fs) xattrIsNotSupported(err error) bool {
 	// Xattrs not supported can be ENOTSUP or ENOATTR or EINVAL (on Solaris)
 	if xattrErr.Err == syscall.EINVAL || xattrErr.Err == syscall.ENOTSUP || xattrErr.Err == xattr.ENOATTR {
 		// Show xattrs not supported
-		if f.xattrSupported.CompareAndSwap(1, 0) {
+		if atomic.CompareAndSwapInt32(&f.xattrSupported, 1, 0) {
 			fs.Errorf(f, "xattrs not supported - disabling: %v", err)
 		}
 		return true
@@ -40,7 +41,7 @@ func (f *Fs) xattrIsNotSupported(err error) bool {
 // It doesn't return any attributes owned by this backend in
 // metadataKeys
 func (o *Object) getXattr() (metadata fs.Metadata, err error) {
-	if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
 		return nil, nil
 	}
 	var list []string
@@ -89,7 +90,7 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) {
 //
 // It doesn't set any attributes owned by this backend in metadataKeys
 func (o *Object) setXattr(metadata fs.Metadata) (err error) {
-	if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
 		return nil
 	}
 	for k, value := range metadata {

backend/mailru/mailru.go

@@ -85,11 +85,10 @@ func init() {
 		Name:        "mailru",
 		Description: "Mail.ru Cloud",
 		NewFs:       NewFs,
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
 			Name:     "user",
 			Help:     "User name (usually email).",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name: "pass",
 			Help: `Password.
@@ -214,7 +213,7 @@ Supported quirks: atomicmkdir binlist unknowndirs`,
 				encoder.EncodeWin | // :?"*<>|
 				encoder.EncodeBackSlash |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 

backend/mega/mega.go

@@ -61,7 +61,6 @@ func init() {
 			Name:     "user",
 			Help:     "User name.",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:       "pass",
 			Help:       "Password.",
@@ -84,17 +83,6 @@ than permanently deleting them.  If you specify this then rclone will
 permanently delete objects instead.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "use_https",
-			Help: `Use HTTPS for transfers.
-
-MEGA uses plain text HTTP connections by default.
-Some ISPs throttle HTTP connections, this causes transfers to become very slow.
-Enabling this will force MEGA to use HTTPS for all transfers.
-HTTPS is normally not necessary since all data is already encrypted anyway.
-Enabling it will increase CPU usage and add network overhead.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -112,7 +100,6 @@ type Options struct {
 	Pass       string               `config:"pass"`
 	Debug      bool                 `config:"debug"`
 	HardDelete bool                 `config:"hard_delete"`
-	UseHTTPS   bool                 `config:"use_https"`
 	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -206,7 +193,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	ci := fs.GetConfig(ctx)
 
-	// cache *mega.Mega on username so we can reuse and share
+	// cache *mega.Mega on username so we can re-use and share
 	// them between remotes.  They are expensive to make as they
 	// contain all the objects and sharing the objects makes the
 	// move code easier as we don't have to worry about mixing
@@ -217,7 +204,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if srv == nil {
 		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
-		srv.SetHTTPS(opt.UseHTTPS)
 		srv.SetLogger(func(format string, v ...interface{}) {
 			fs.Infof("*go-mega*", format, v...)
 		})

backend/netstorage/netstorage.go

@@ -66,12 +66,10 @@ HTTP is provided primarily for debugging purposes.`,
 
 Format should be ` + "`<domain>/<internal folders>`",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name:     "account",
 			Help:     "Set the NetStorage account name",
 			Required: true,
-			Sensitive: true,
 		}, {
 			Name: "secret",
 			Help: `Set the NetStorage account secret/G2O key for authentication.
@@ -821,8 +819,6 @@ func (f *Fs) getAuth(req *http.Request) error {
 	// Set Authorization header
 	dataHeader := generateDataHeader(f)
 	path := req.URL.RequestURI()
-	//lint:ignore SA1008 false positive when running staticcheck, the header name is according to docs even if not canonical
-	//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1008
 	actionHeader := req.Header["X-Akamai-ACS-Action"][0]
 	fs.Debugf(nil, "NetStorage API %s call %s for path %q", req.Method, actionHeader, path)
 	req.Header.Set("X-Akamai-ACS-Auth-Data", dataHeader)

backend/onedrive/api/types.go

@@ -99,16 +99,6 @@ type ItemReference struct {
 	DriveType string `json:"driveType"` // Type of the drive,	Read-Only
 }
 
-// GetID returns a normalized ID of the item
-// If DriveID is known it will be prefixed to the ID with # separator
-// Can be parsed using onedrive.parseNormalizedID(normalizedID)
-func (i *ItemReference) GetID() string {
-	if !strings.Contains(i.ID, "#") {
-		return i.DriveID + "#" + i.ID
-	}
-	return i.ID
-}
-
 // RemoteItemFacet groups data needed to reference a OneDrive remote item
 type RemoteItemFacet struct {
 	ID                   string               `json:"id"`                   // The unique identifier of the item within the remote Drive. Read-only.
@@ -136,7 +126,6 @@ type HashesType struct {
 	Sha1Hash     string `json:"sha1Hash"`     // hex encoded SHA1 hash for the contents of the file (if available)
 	Crc32Hash    string `json:"crc32Hash"`    // hex encoded CRC32 value of the file (if available)
 	QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available)
-	Sha256Hash   string `json:"sha256Hash"`   // hex encoded SHA256 value of the file (if available)
 }
 
 // FileFacet groups file-related data on OneDrive into a single structure.
@@ -195,8 +184,8 @@ type Item struct {
 	Deleted *DeletedFacet `json:"deleted"` // Information about the deleted state of the item. Read-only.
 }
 
-// DeltaResponse is the response to the view delta method
-type DeltaResponse struct {
+// ViewDeltaResponse is the response to the view delta method
+type ViewDeltaResponse struct {
 	Value      []Item `json:"value"`            // An array of Item objects which have been created, modified, or deleted.
 	NextLink   string `json:"@odata.nextLink"`  // A URL to retrieve the next available page of changes.
 	DeltaLink  string `json:"@odata.deltaLink"` // A URL returned instead of @odata.nextLink after all current changes have been returned. Used to read the next set of changes in the future.
@@ -448,27 +437,3 @@ type Version struct {
 type VersionsResponse struct {
 	Versions []Version `json:"value"`
 }
-
-// DriveResource is returned from /me/drive
-type DriveResource struct {
-	DriveID   string `json:"id"`
-	DriveName string `json:"name"`
-	DriveType string `json:"driveType"`
-}
-
-// DrivesResponse is returned from /sites/{siteID}/drives",
-type DrivesResponse struct {
-	Drives []DriveResource `json:"value"`
-}
-
-// SiteResource is part of the response from from "/sites/root:"
-type SiteResource struct {
-	SiteID   string `json:"id"`
-	SiteName string `json:"displayName"`
-	SiteURL  string `json:"webUrl"`
-}
-
-// SiteResponse is returned from "/sites/root:"
-type SiteResponse struct {
-	Sites []SiteResource `json:"value"`
-}

backend/onedrive/onedrive.go

@@ -27,7 +27,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
@@ -136,7 +135,6 @@ Note that the chunks will be buffered into memory.`,
 			Help:     "The ID of the drive to use.",
 			Default:  "",
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name:     "drive_type",
 			Help:     "The type of the drive (" + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + ").",
@@ -151,7 +149,6 @@ know the folder ID that you wish to access but not be able to get
 there through a path traversal.
 `,
 			Advanced: true,
-			Sensitive: true,
 		}, {
 			Name: "access_scopes",
 			Help: `Set scopes to be requested by rclone.
@@ -199,9 +196,7 @@ listing, set this option.`,
 		}, {
 			Name:    "server_side_across_configs",
 			Default: false,
-			Help: `Deprecated: use --server-side-across-configs instead.
-
-Allow server-side operations (e.g. copy) to work across different onedrive configs.
+			Help: `Allow server-side operations (e.g. copy) to work across different onedrive configs.
 
 This will only work if you are copying between two OneDrive *Personal* drives AND
 the files to copy are already shared between them.  In other cases, rclone will
@@ -264,98 +259,6 @@ this flag there.
 At the time of writing this only works with OneDrive personal paid accounts.
 `,
 			Advanced: true,
-			Sensitive: true,
-		}, {
-			Name:    "hash_type",
-			Default: "auto",
-			Help: `Specify the hash in use for the backend.
-
-This specifies the hash type in use. If set to "auto" it will use the
-default hash which is QuickXorHash.
-
-Before rclone 1.62 an SHA1 hash was used by default for Onedrive
-Personal. For 1.62 and later the default is to use a QuickXorHash for
-all onedrive types. If an SHA1 hash is desired then set this option
-accordingly.
-
-From July 2023 QuickXorHash will be the only available hash for
-both OneDrive for Business and OneDriver Personal.
-
-This can be set to "none" to not use any hashes.
-
-If the hash requested does not exist on the object, it will be
-returned as an empty string which is treated as a missing hash by
-rclone.
-`,
-			Examples: []fs.OptionExample{{
-				Value: "auto",
-				Help:  "Rclone chooses the best hash",
-			}, {
-				Value: "quickxor",
-				Help:  "QuickXor",
-			}, {
-				Value: "sha1",
-				Help:  "SHA1",
-			}, {
-				Value: "sha256",
-				Help:  "SHA256",
-			}, {
-				Value: "crc32",
-				Help:  "CRC32",
-			}, {
-				Value: "none",
-				Help:  "None - don't use any hashes",
-			}},
-			Advanced: true,
-		}, {
-			Name:    "av_override",
-			Default: false,
-			Help: `Allows download of files the server thinks has a virus.
-
-The onedrive/sharepoint server may check files uploaded with an Anti
-Virus checker. If it detects any potential viruses or malware it will
-block download of the file.
-
-In this case you will see a message like this
-
-    server reports this file is infected with a virus - use --onedrive-av-override to download anyway: Infected (name of virus): 403 Forbidden: 
-
-If you are 100% sure you want to download this file anyway then use
-the --onedrive-av-override flag, or av_override = true in the config
-file.
-`,
-			Advanced: true,
-		}, {
-			Name:    "delta",
-			Default: false,
-			Help: strings.ReplaceAll(`If set rclone will use delta listing to implement recursive listings.
-
-If this flag is set the the onedrive backend will advertise |ListR|
-support for recursive listings.
-
-Setting this flag speeds up these things greatly:
-
-    rclone lsf -R onedrive:
-    rclone size onedrive:
-    rclone rc vfs/refresh recursive=true
-
-**However** the delta listing API **only** works at the root of the
-drive. If you use it not at the root then it recurses from the root
-and discards all the data that is not under the directory you asked
-for. So it will be correct but may not be very efficient.
-
-This is why this flag is not set as the default.
-
-As a rule of thumb if nearly all of your data is under rclone's root
-directory (the |root/directory| in |onedrive:root/directory|) then
-using this flag will be be a big performance win. If your data is
-mostly not under the root then using this flag will be a big
-performance loss.
-
-It is recommended if you are mounting your onedrive at the root
-(or near the root when using crypt) and using rclone |rc vfs/refresh|.
-`, "|", "`"),
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -403,6 +306,28 @@ It is recommended if you are mounting your onedrive at the root
 	})
 }
 
+type driveResource struct {
+	DriveID   string `json:"id"`
+	DriveName string `json:"name"`
+	DriveType string `json:"driveType"`
+}
+type drivesResponse struct {
+	Drives []driveResource `json:"value"`
+}
+
+type siteResource struct {
+	SiteID   string `json:"id"`
+	SiteName string `json:"displayName"`
+	SiteURL  string `json:"webUrl"`
+}
+type siteResponse struct {
+	Sites []siteResource `json:"value"`
+}
+type deltaResponse struct {
+	DeltaLink string     `json:"@odata.deltaLink"`
+	Value     []api.Item `json:"value"`
+}
+
 // Get the region and graphURL from the config
 func getRegionURL(m configmap.Mapper) (region, graphURL string) {
 	region, _ = m.Get("region")
@@ -429,7 +354,7 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 			RootURL: graphURL,
 			Path:    "/sites/root:" + opt.relativePath,
 		}
-		site := api.SiteResource{}
+		site := siteResource{}
 		_, err := srv.CallJSON(ctx, &opt.opts, nil, &site)
 		if err != nil {
 			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available site by relative path: %v", err))
@@ -446,7 +371,7 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 		}
 	}
 
-	drives := api.DrivesResponse{}
+	drives := drivesResponse{}
 
 	// We don't have the final ID yet?
 	// query Microsoft Graph
@@ -459,7 +384,7 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 		// Also call /me/drive as sometimes /me/drives doesn't return it #4068
 		if opt.opts.Path == "/me/drives" {
 			opt.opts.Path = "/me/drive"
-			meDrive := api.DriveResource{}
+			meDrive := driveResource{}
 			_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)
 			if err != nil {
 				return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
@@ -478,7 +403,7 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 			}
 		}
 	} else {
-		drives.Drives = append(drives.Drives, api.DriveResource{
+		drives.Drives = append(drives.Drives, driveResource{
 			DriveID:   opt.finalDriveID,
 			DriveName: "Chosen Drive ID",
 			DriveType: "drive",
@@ -582,18 +507,15 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 	case "url":
 		return fs.ConfigInput("url_end", "config_site_url", `Site URL
 
-Examples:
-- "mysite"
-- "https://XXX.sharepoint.com/sites/mysite"
-- "https://XXX.sharepoint.com/teams/ID"
+Example: "https://contoso.sharepoint.com/sites/mysite" or "mysite"
 `)
 	case "url_end":
 		siteURL := config.Result
-		re := regexp.MustCompile(`https://.*\.sharepoint\.com(/.*)`)
+		re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
 		match := re.FindStringSubmatch(siteURL)
 		if len(match) == 2 {
 			return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-				relativePath: match[1],
+				relativePath: "/sites/" + match[1],
 			})
 		}
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
@@ -615,7 +537,7 @@ Examples:
 			Path:    "/sites?search=" + searchTerm,
 		}
 
-		sites := api.SiteResponse{}
+		sites := siteResponse{}
 		_, err := srv.CallJSON(ctx, &opts, nil, &sites)
 		if err != nil {
 			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available sites: %v", err))
@@ -675,9 +597,6 @@ type Options struct {
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
 	LinkPassword            string               `config:"link_password"`
-	HashType                string               `config:"hash_type"`
-	AVOverride              bool                 `config:"av_override"`
-	Delta                   bool                 `config:"delta"`
 	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -689,13 +608,11 @@ type Fs struct {
 	ci           *fs.ConfigInfo     // global config
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the OneDrive server
-	unAuth       *rest.Client       // no authentication connection to the OneDrive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
 	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
-	hashType     hash.Type          // type of the hash we are using
 }
 
 // Object describes a OneDrive object
@@ -709,7 +626,8 @@ type Object struct {
 	size          int64     // size of the object
 	modTime       time.Time // modification time of the object
 	id            string    // ID of the object
-	hash          string    // Hash of the content, usually QuickXorHash but set as hash_type
+	sha1          string    // SHA-1 of the object content
+	quickxorhash  string    // QuickXorHash of the object content
 	mimeType      string    // Content-Type of object from server (may not be as uploaded)
 }
 
@@ -948,9 +866,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		TokenURL: authEndpoint[opt.Region] + tokenPath,
 	}
 
-	client := fshttp.NewClient(ctx)
 	root = parsePath(root)
-	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, client)
+	oAuthClient, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure OneDrive: %w", err)
 	}
@@ -964,9 +881,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		driveID:   opt.DriveID,
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
-		unAuth:    rest.NewClient(client).SetRoot(rootURL),
 		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		hashType:  QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -976,15 +891,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Set the user defined hash
-	if opt.HashType == "auto" || opt.HashType == "" {
-		opt.HashType = QuickXorHashType.String()
-	}
-	err = f.hashType.Set(opt.HashType)
-	if err != nil {
-		return nil, err
-	}
-
 	// Disable change polling in China region
 	// See: https://github.com/rclone/rclone/issues/6444
 	if f.opt.Region == regionCN {
@@ -1012,11 +918,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	f.dirCache = dircache.New(root, rootID, f)
 
-	// ListR only supported if delta set
-	if !f.opt.Delta {
-		f.features.ListR = nil
-	}
-
 	// Find the current root
 	err = f.dirCache.FindRoot(ctx, false)
 	if err != nil {
@@ -1136,29 +1037,32 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
 // If directories is set it only sends directories
 // User function to process a File item from listAll
 //
-// If an error is returned then processing stops
-type listAllFn func(*api.Item) error
+// Should return true to finish processing
+type listAllFn func(*api.Item) bool
 
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-//
-// This listing function works on both normal listings and delta listings
-func (f *Fs) _listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn, opts *rest.Opts, result any, pValue *[]api.Item, pNextLink *string) (err error) {
+func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+	// Top parameter asks for bigger pages of data
+	// https://dev.onedrive.com/odata/optional-query-parameters.htm
+	opts := f.newOptsCall(dirID, "GET", fmt.Sprintf("/children?$top=%d", f.opt.ListChunk))
+OUTER:
 	for {
+		var result api.ListChildrenResponse
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(ctx, opts, nil, result)
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 			return shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
-			return fmt.Errorf("couldn't list files: %w", err)
+			return found, fmt.Errorf("couldn't list files: %w", err)
 		}
-		if len(*pValue) == 0 {
+		if len(result.Value) == 0 {
 			break
 		}
-		for i := range *pValue {
-			item := &(*pValue)[i]
+		for i := range result.Value {
+			item := &result.Value[i]
 			isFolder := item.GetFolder() != nil
 			if isFolder {
 				if filesOnly {
@@ -1173,60 +1077,18 @@ func (f *Fs) _listAll(ctx context.Context, dirID string, directoriesOnly bool, f
 				continue
 			}
 			item.Name = f.opt.Enc.ToStandardName(item.GetName())
-			err = fn(item)
-			if err != nil {
-				return err
+			if fn(item) {
+				found = true
+				break OUTER
 			}
 		}
-		if *pNextLink == "" {
+		if result.NextLink == "" {
 			break
 		}
 		opts.Path = ""
-		opts.Parameters = nil
-		opts.RootURL = *pNextLink
-		// reset results
-		*pNextLink = ""
-		*pValue = nil
+		opts.RootURL = result.NextLink
 	}
-	return nil
-}
-
-// Lists the directory required calling the user function on each item found
-//
-// If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (err error) {
-	// Top parameter asks for bigger pages of data
-	// https://dev.onedrive.com/odata/optional-query-parameters.htm
-	opts := f.newOptsCall(dirID, "GET", fmt.Sprintf("/children?$top=%d", f.opt.ListChunk))
-	var result api.ListChildrenResponse
-	return f._listAll(ctx, dirID, directoriesOnly, filesOnly, fn, &opts, &result, &result.Value, &result.NextLink)
-}
-
-// Convert a list item into a DirEntry
-//
-// Can return nil for an item which should be skipped
-func (f *Fs) itemToDirEntry(ctx context.Context, dir string, info *api.Item) (entry fs.DirEntry, err error) {
-	if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote {
-		fs.Debugf(info.Name, "OneNote file not shown in directory listing")
-		return nil, nil
-	}
-	remote := path.Join(dir, info.GetName())
-	folder := info.GetFolder()
-	if folder != nil {
-		// cache the directory ID for later lookups
-		id := info.GetID()
-		f.dirCache.Put(remote, id)
-		d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-		d.SetItems(folder.ChildCount)
-		entry = d
-	} else {
-		o, err := f.newObjectWithInfo(ctx, remote, info)
-		if err != nil {
-			return nil, err
-		}
-		entry = o
-	}
-	return entry, nil
+	return
 }
 
 // List the objects and directories in dir into entries.  The
@@ -1243,144 +1105,41 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if err != nil {
 		return nil, err
 	}
-	err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) error {
-		entry, err := f.itemToDirEntry(ctx, dir, info)
+	var iErr error
+	_, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
+		if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote {
+			fs.Debugf(info.Name, "OneNote file not shown in directory listing")
+			return false
+		}
+
+		remote := path.Join(dir, info.GetName())
+		folder := info.GetFolder()
+		if folder != nil {
+			// cache the directory ID for later lookups
+			id := info.GetID()
+			f.dirCache.Put(remote, id)
+			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
+			d.SetItems(folder.ChildCount)
+			entries = append(entries, d)
+		} else {
+			o, err := f.newObjectWithInfo(ctx, remote, info)
 			if err != nil {
-			return err
+				iErr = err
+				return true
 			}
-		if entry == nil {
-			return nil
+			entries = append(entries, o)
 		}
-		entries = append(entries, entry)
-		return nil
+		return false
 	})
 	if err != nil {
 		return nil, err
 	}
+	if iErr != nil {
+		return nil, iErr
+	}
 	return entries, nil
 }
 
-// ListR lists the objects and directories of the Fs starting
-// from dir recursively into out.
-//
-// dir should be "" to start from the root, and should not
-// have trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-//
-// It should call callback for each tranche of entries read.
-// These need not be returned in any particular order.  If
-// callback returns an error then the listing will stop
-// immediately.
-//
-// Don't implement this unless you have a more efficient way
-// of listing recursively than doing a directory traversal.
-func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	// Make sure this ID is in the directory cache
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-
-	// ListR only works at the root of a onedrive, not on a folder
-	// So we have to filter things outside of the root which is
-	// inefficient.
-
-	list := walk.NewListRHelper(callback)
-
-	// list a folder conventionally - used for shared folders
-	var listFolder func(dir string) error
-	listFolder = func(dir string) error {
-		entries, err := f.List(ctx, dir)
-		if err != nil {
-			return err
-		}
-		for _, entry := range entries {
-			err = list.Add(entry)
-			if err != nil {
-				return err
-			}
-			if _, isDir := entry.(fs.Directory); isDir {
-				err = listFolder(entry.Remote())
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	}
-
-	// This code relies on the fact that directories are sent before their children. This isn't
-	// mentioned in the docs though, so maybe it shouldn't be relied on.
-	seen := map[string]struct{}{}
-	fn := func(info *api.Item) error {
-		var parentPath string
-		var ok bool
-		id := info.GetID()
-		// The API can produce duplicates, so skip them
-		if _, found := seen[id]; found {
-			return nil
-		}
-		seen[id] = struct{}{}
-		// Skip the root directory
-		if id == directoryID {
-			return nil
-		}
-		// Skip deleted items
-		if info.Deleted != nil {
-			return nil
-		}
-		dirID := info.GetParentReference().GetID()
-		// Skip files that don't have their parent directory
-		// cached as they are outside the root.
-		parentPath, ok = f.dirCache.GetInv(dirID)
-		if !ok {
-			return nil
-		}
-		// Skip files not under the root directory
-		remote := path.Join(parentPath, info.GetName())
-		if dir != "" && !strings.HasPrefix(remote, dir+"/") {
-			return nil
-		}
-		entry, err := f.itemToDirEntry(ctx, parentPath, info)
-		if err != nil {
-			return err
-		}
-		if entry == nil {
-			return nil
-		}
-		err = list.Add(entry)
-		if err != nil {
-			return err
-		}
-		// If this is a shared folder, we'll need list it too
-		if info.RemoteItem != nil && info.RemoteItem.Folder != nil {
-			fs.Debugf(remote, "Listing shared directory")
-			return listFolder(remote)
-		}
-		return nil
-	}
-
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   "/root/delta",
-		Parameters: map[string][]string{
-			// "token": {token},
-			"$top": {fmt.Sprintf("%d", f.opt.ListChunk)},
-		},
-	}
-
-	var result api.DeltaResponse
-	err = f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
-	if err != nil {
-		return err
-	}
-
-	return list.Flush()
-
-}
-
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
@@ -1449,12 +1208,15 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	}
 	if check {
 		// check to see if there are any items
-		err := f.listAll(ctx, rootID, false, false, func(item *api.Item) error {
-			return fs.ErrorDirectoryNotEmpty
+		found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool {
+			return true
 		})
 		if err != nil {
 			return err
 		}
+		if found {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 	err = f.deleteObject(ctx, rootID)
 	if err != nil {
@@ -1794,7 +1556,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(f.hashType)
+	if f.driveType == driveTypePersonal {
+		return hash.Set(hash.SHA1)
+	}
+	return hash.Set(QuickXorHashType)
 }
 
 // PublicLink returns a link for downloading without account.
@@ -1909,10 +1674,6 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	token := make(chan struct{}, f.ci.Checkers)
 	var wg sync.WaitGroup
 	err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
-		if err != nil {
-			fs.Errorf(f, "Failed to list %q: %v", path, err)
-			return nil
-		}
 		err = entries.ForObjectError(func(obj fs.Object) error {
 			o, ok := obj.(*Object)
 			if !ok {
@@ -2007,8 +1768,14 @@ func (o *Object) rootPath() string {
 
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == o.fs.hashType {
-		return o.hash, nil
+	if o.fs.driveType == driveTypePersonal {
+		if t == hash.SHA1 {
+			return o.sha1, nil
+		}
+	} else {
+		if t == QuickXorHashType {
+			return o.quickxorhash, nil
+		}
 	}
 	return "", hash.ErrUnsupported
 }
@@ -2039,24 +1806,17 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	file := info.GetFile()
 	if file != nil {
 		o.mimeType = file.MimeType
-		o.hash = ""
-		switch o.fs.hashType {
-		case QuickXorHashType:
+		if file.Hashes.Sha1Hash != "" {
+			o.sha1 = strings.ToLower(file.Hashes.Sha1Hash)
+		}
 		if file.Hashes.QuickXorHash != "" {
 			h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
 			if err != nil {
 				fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
 			} else {
-					o.hash = hex.EncodeToString(h)
+				o.quickxorhash = hex.EncodeToString(h)
 			}
 		}
-		case hash.SHA1:
-			o.hash = strings.ToLower(file.Hashes.Sha1Hash)
-		case hash.SHA256:
-			o.hash = strings.ToLower(file.Hashes.Sha256Hash)
-		case hash.CRC32:
-			o.hash = strings.ToLower(file.Hashes.Crc32Hash)
-		}
 	}
 	fileSystemInfo := info.GetFileSystemInfo()
 	if fileSystemInfo != nil {
@@ -2151,20 +1911,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var resp *http.Response
 	opts := o.fs.newOptsCall(o.id, "GET", "/content")
 	opts.Options = options
-	if o.fs.opt.AVOverride {
-		opts.Parameters = url.Values{"AVOverride": {"1"}}
-	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		if resp != nil {
-			if virus := resp.Header.Get("X-Virus-Infected"); virus != "" {
-				err = fmt.Errorf("server reports this file is infected with a virus - use --onedrive-av-override to download anyway: %s: %w", virus, err)
-			}
-		}
 		return nil, err
 	}
 
@@ -2243,7 +1995,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			Options:       options,
 		}
 		_, _ = chunk.Seek(skip, io.SeekStart)
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
 			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
 			pos, posErr := o.getPosition(ctx, url)
@@ -2639,7 +2391,7 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
 	return
 }
 
-func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
+func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta deltaResponse, err error) {
 	opts := f.buildDriveDeltaOpts(token)
 
 	_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
@@ -2758,7 +2510,6 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
-	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
 	_ fs.IDer            = &Object{}

backend/onedrive/quickxorhash/quickxorhash.go

@@ -7,40 +7,51 @@
 // See: https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
 package quickxorhash
 
-// This code was ported from a fast C-implementation from
-// https://github.com/namazso/QuickXorHash
-// which has licenced as BSD Zero Clause License
-//
-// BSD Zero Clause License
-//
-// Copyright (c) 2022 namazso <admin@namazso.eu>
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
+// This code was ported from the code snippet linked from
+// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+// Which has the copyright
 
-import "hash"
+// ------------------------------------------------------------------------------
+//  Copyright (c) 2016 Microsoft Corporation
+//
+//  Permission is hereby granted, free of charge, to any person obtaining a copy
+//  of this software and associated documentation files (the "Software"), to deal
+//  in the Software without restriction, including without limitation the rights
+//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the Software is
+//  furnished to do so, subject to the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included in
+//  all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+//  THE SOFTWARE.
+// ------------------------------------------------------------------------------
+
+import (
+	"hash"
+)
 
 const (
 	// BlockSize is the preferred size for hashing
 	BlockSize = 64
 	// Size of the output checksum
 	Size           = 20
+	bitsInLastCell = 32
 	shift          = 11
 	widthInBits    = 8 * Size
-	dataSize    = shift * widthInBits
+	dataSize       = (widthInBits-1)/64 + 1
 )
 
 type quickXorHash struct {
-	data [dataSize]byte
-	size uint64
+	data        [dataSize]uint64
+	lengthSoFar uint64
+	shiftSoFar  int
 }
 
 // New returns a new hash.Hash computing the quickXorHash checksum.
@@ -59,37 +70,94 @@ func New() hash.Hash {
 //
 // Implementations must not retain p.
 func (q *quickXorHash) Write(p []byte) (n int, err error) {
-	var i int
-	// fill last remain
-	lastRemain := q.size % dataSize
-	if lastRemain != 0 {
-		i += xorBytes(q.data[lastRemain:], p)
+	currentshift := q.shiftSoFar
+
+	// The bitvector where we'll start xoring
+	vectorArrayIndex := currentshift / 64
+
+	// The position within the bit vector at which we begin xoring
+	vectorOffset := currentshift % 64
+	iterations := len(p)
+	if iterations > widthInBits {
+		iterations = widthInBits
 	}
 
-	if i != len(p) {
-		for len(p)-i >= dataSize {
-			i += xorBytes(q.data[:], p[i:])
+	for i := 0; i < iterations; i++ {
+		isLastCell := vectorArrayIndex == len(q.data)-1
+		var bitsInVectorCell int
+		if isLastCell {
+			bitsInVectorCell = bitsInLastCell
+		} else {
+			bitsInVectorCell = 64
 		}
-		xorBytes(q.data[:], p[i:])
+
+		// There's at least 2 bitvectors before we reach the end of the array
+		if vectorOffset <= bitsInVectorCell-8 {
+			for j := i; j < len(p); j += widthInBits {
+				q.data[vectorArrayIndex] ^= uint64(p[j]) << uint(vectorOffset)
 			}
-	q.size += uint64(len(p))
+		} else {
+			index1 := vectorArrayIndex
+			var index2 int
+			if isLastCell {
+				index2 = 0
+			} else {
+				index2 = vectorArrayIndex + 1
+			}
+			low := byte(bitsInVectorCell - vectorOffset)
+
+			xoredByte := byte(0)
+			for j := i; j < len(p); j += widthInBits {
+				xoredByte ^= p[j]
+			}
+			q.data[index1] ^= uint64(xoredByte) << uint(vectorOffset)
+			q.data[index2] ^= uint64(xoredByte) >> low
+		}
+		vectorOffset += shift
+		for vectorOffset >= bitsInVectorCell {
+			if isLastCell {
+				vectorArrayIndex = 0
+			} else {
+				vectorArrayIndex = vectorArrayIndex + 1
+			}
+			vectorOffset -= bitsInVectorCell
+		}
+	}
+
+	// Update the starting position in a circular shift pattern
+	q.shiftSoFar = (q.shiftSoFar + shift*(len(p)%widthInBits)) % widthInBits
+
+	q.lengthSoFar += uint64(len(p))
+
 	return len(p), nil
 }
 
 // Calculate the current checksum
-func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
-	for i := 0; i < dataSize; i++ {
-		shift := (i * 11) % 160
-		shiftBytes := shift / 8
-		shiftBits := shift % 8
-		shifted := int(q.data[i]) << shiftBits
-		h[shiftBytes] ^= byte(shifted)
-		h[shiftBytes+1] ^= byte(shifted >> 8)
+func (q *quickXorHash) checkSum() (h [Size]byte) {
+	// Output the data as little endian bytes
+	ph := 0
+	for i := 0; i < len(q.data)-1; i++ {
+		d := q.data[i]
+		_ = h[ph+7] // bounds check
+		h[ph+0] = byte(d >> (8 * 0))
+		h[ph+1] = byte(d >> (8 * 1))
+		h[ph+2] = byte(d >> (8 * 2))
+		h[ph+3] = byte(d >> (8 * 3))
+		h[ph+4] = byte(d >> (8 * 4))
+		h[ph+5] = byte(d >> (8 * 5))
+		h[ph+6] = byte(d >> (8 * 6))
+		h[ph+7] = byte(d >> (8 * 7))
+		ph += 8
 	}
-	h[0] ^= h[20]
+	// remaining 32 bits
+	d := q.data[len(q.data)-1]
+	h[Size-4] = byte(d >> (8 * 0))
+	h[Size-3] = byte(d >> (8 * 1))
+	h[Size-2] = byte(d >> (8 * 2))
+	h[Size-1] = byte(d >> (8 * 3))
 
 	// XOR the file length with the least significant bits in little endian format
-	d := q.size
+	d = q.lengthSoFar
 	h[Size-8] ^= byte(d >> (8 * 0))
 	h[Size-7] ^= byte(d >> (8 * 1))
 	h[Size-6] ^= byte(d >> (8 * 2))
@@ -106,7 +174,7 @@ func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
 // It does not change the underlying hash state.
 func (q *quickXorHash) Sum(b []byte) []byte {
 	hash := q.checkSum()
-	return append(b, hash[:Size]...)
+	return append(b, hash[:]...)
 }
 
 // Reset resets the Hash to its initial state.
@@ -128,10 +196,8 @@ func (q *quickXorHash) BlockSize() int {
 }
 
 // Sum returns the quickXorHash checksum of the data.
-func Sum(data []byte) (h [Size]byte) {
+func Sum(data []byte) [Size]byte {
 	var d quickXorHash
 	_, _ = d.Write(data)
-	s := d.checkSum()
-	copy(h[:], s[:])
-	return h
+	return d.checkSum()
 }