vfs: disable --buffer-size read ahead

See: https://forum.rclone.org/t/for-mount-sftp-why-right-click-on-exe-file-is-so-slow-until-it-freezes/33830/52

.github/workflows/build.yml

@@ -25,12 +25,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.16', 'go1.17']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.19'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -39,9 +39,16 @@ jobs:
             librclonetest: true
             deploy: true
 
+          - job_name: linux_386
+            os: ubuntu-latest
+            go: '1.19'
+            goarch: 386
+            gotags: cmount
+            quicktest: true
+
           - job_name: mac_amd64
             os: macos-11
-            go: '1.18.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -50,48 +57,37 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.18.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
-          - job_name: windows_amd64
+          - job_name: windows
             os: windows-latest
-            go: '1.18.x'
+            go: '1.19'
             gotags: cmount
-            build_flags: '-include "^windows/amd64" -cgo'
-            build_args: '-buildmode exe'
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: windows_386
-            os: windows-latest
-            go: '1.18.x'
-            gotags: cmount
-            goarch: '386'
-            cgo: '1'
-            build_flags: '-include "^windows/386" -cgo'
+            cgo: '0'
+            build_flags: '-include "^windows/"'
             build_args: '-buildmode exe'
             quicktest: true
             deploy: true
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.18.x'
-            build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
+            go: '1.19'
+            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.16
+          - job_name: go1.17
             os: ubuntu-latest
-            go: '1.16.x'
+            go: '1.17'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.17
+          - job_name: go1.18
             os: ubuntu-latest
-            go: '1.17.x'
+            go: '1.18'
             quicktest: true
             racequicktest: true
 
@@ -101,14 +97,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 
@@ -166,7 +161,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -178,6 +173,11 @@ jobs:
         run: |
           make
 
+      - name: Rclone version
+        shell: bash
+        run: |
+          rclone version
+
       - name: Run tests
         shell: bash
         run: |
@@ -225,14 +225,27 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Code quality test
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
+      # Run govulncheck on the latest go version, the one we build binaries with
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+          check-latest: true
+
+      - name: Install govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+
+      - name: Scan for vulnerabilities
+        run: govulncheck ./...
+
   android:
     if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
@@ -241,22 +254,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.18.x
-
-      # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
-      - name: Force NDK version
-        run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;23.1.7779620" | grep -v = || true
+          go-version: 1.19
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -277,27 +286,29 @@ jobs:
           go install golang.org/x/mobile/cmd/gobind@latest
           go install golang.org/x/mobile/cmd/gomobile@latest
           env PATH=$PATH:~/go/bin gomobile init
+          echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV
 
       - name: arm-v7a gomobile build
-        run: env PATH=$PATH:~/go/bin gomobile bind -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
+        run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
 
       - name: arm-v7a Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=arm' >> $GITHUB_ENV
           echo 'GOARM=7' >> $GITHUB_ENV
           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
+
       - name: arm-v7a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
 
       - name: arm64-v8a Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=arm64' >> $GITHUB_ENV
@@ -305,12 +316,12 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: arm64-v8a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
 
       - name: x86 Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=386' >> $GITHUB_ENV
@@ -318,12 +329,12 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: x86 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
 
       - name: x64 Set environment variables
         shell: bash
         run: |
-          echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
+          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
           echo 'GOOS=android' >> $GITHUB_ENV
           echo 'GOARCH=amd64' >> $GITHUB_ENV
@@ -331,7 +342,7 @@ jobs:
           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
 
       - name: x64 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
+        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 .
 
       - name: Upload artifacts
         run: |

.github/workflows/build_publish_docker_image.yml

@@ -12,7 +12,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish image
@@ -20,7 +20,7 @@ jobs:
               with:
                 tag: beta
                 imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
                 publish: true
                 dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
                 dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -28,7 +28,7 @@ jobs:
               with:
                 tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
                 imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
                 publish: true
                 dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
                 dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin
@@ -50,7 +50,7 @@ jobs:
                 PLUGIN_USER=rclone
                 docker login --username ${{ secrets.DOCKER_HUB_USER }} \
                              --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
                     export PLUGIN_USER PLUGIN_ARCH
                     make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
                     make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}

.golangci.yml

@@ -20,7 +20,7 @@ issues:
   exclude-use-default: false
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
+  max-issues-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0

CONTRIBUTING.md

@@ -77,7 +77,7 @@ Make sure you
   * Add [documentation](#writing-documentation) for a new feature.
   * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
 
-When you are done with that push your changes to Github:
+When you are done with that push your changes to GitHub:
 
     git push -u origin my-new-feature
 
@@ -88,7 +88,7 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I
 
 You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
 
-## Using Git and Github ##
+## Using Git and GitHub ##
 
 ### Committing your changes ###
 

Makefile

@@ -97,7 +97,7 @@ release_dep_linux:
 
 # Get the release dependencies we only install on Windows
 release_dep_windows:
-	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
+	GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
 
 # Update dependencies
 showupdates:
@@ -245,18 +245,18 @@ retag:
 startdev:
 	@echo "Version is $(VERSION)"
 	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
 	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
 	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
 startstable:
 	@echo "Version is $(VERSION)"
 	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
 	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
 	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -29,6 +29,8 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
+  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
+  * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
   * Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
@@ -40,10 +42,13 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
+  * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
-  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
+  * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
+  * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
@@ -57,17 +62,20 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
+  * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
+  * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * Storj [:page_facing_up:](https://rclone.org/storj/)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
@@ -80,6 +88,19 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
 
 Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
 
+### Virtual storage providers
+
+These backends adapt or modify other storage providers
+
+  * Alias: rename existing remotes [:page_facing_up:](https://rclone.org/alias/)
+  * Cache: cache remotes (DEPRECATED) [:page_facing_up:](https://rclone.org/cache/)
+  * Chunker: split large files [:page_facing_up:](https://rclone.org/chunker/)
+  * Combine: combine multiple remotes into a directory tree [:page_facing_up:](https://rclone.org/combine/)
+  * Compress: compress files [:page_facing_up:](https://rclone.org/compress/)
+  * Crypt: encrypt files [:page_facing_up:](https://rclone.org/crypt/)
+  * Hasher: hash files [:page_facing_up:](https://rclone.org/hasher/)
+  * Union: join multiple remotes to work together [:page_facing_up:](https://rclone.org/union/)
+
 ## Features
 
   * MD5/SHA-1 hashes checked at all times for file integrity
@@ -94,7 +115,7 @@ Please see [the full list of all storage providers and their features](https://r
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
   * Multi-threaded downloads to local disk
-  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
+  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDAV/FTP/SFTP/DLNA
 
 ## Installation & documentation
 
@@ -115,5 +136,5 @@ Please see the [rclone website](https://rclone.org/) for:
 License
 -------
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 [COPYING file](/COPYING) included in this package).

RELEASE.md

@@ -53,6 +53,14 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
+## Tidy beta
+
+At some point after the release run
+
+    bin/tidy-beta v1.55
+
+where the version number is that of a couple ago to remove old beta binaries.
+
 ## Making a point release
 
 If rclone needs a point release due to some horrendous bug:

VERSION

@@ -1 +1 @@
-v1.59.0
+v1.61.0

backend/alias/alias.go

@@ -1,3 +1,4 @@
+// Package alias implements a virtual provider to rename existing remotes.
 package alias
 
 import (

backend/all/all.go

@@ -1,3 +1,4 @@
+// Package all imports all the backends
 package all
 
 import (
@@ -9,6 +10,7 @@ import (
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
+	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/drive"
@@ -20,8 +22,9 @@ import (
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
 	_ "github.com/rclone/rclone/backend/hdfs"
+	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/hubic"
+	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
 	_ "github.com/rclone/rclone/backend/local"
@@ -31,6 +34,7 @@ import (
 	_ "github.com/rclone/rclone/backend/netstorage"
 	_ "github.com/rclone/rclone/backend/onedrive"
 	_ "github.com/rclone/rclone/backend/opendrive"
+	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/putio"
@@ -40,6 +44,7 @@ import (
 	_ "github.com/rclone/rclone/backend/sftp"
 	_ "github.com/rclone/rclone/backend/sharefile"
 	_ "github.com/rclone/rclone/backend/sia"
+	_ "github.com/rclone/rclone/backend/smb"
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"

backend/amazonclouddrive/amazonclouddrive.go

@@ -435,7 +435,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, title string, directorie
 		query += " AND kind:" + folderKind
 	} else if filesOnly {
 		query += " AND kind:" + fileKind
-	} else {
+		//} else {
 		// FIXME none of these work
 		//query += " AND kind:(" + fileKind + " OR " + folderKind + ")"
 		//query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")"
@@ -556,9 +556,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // This is a workaround for Amazon sometimes returning
 //
-//  * 408 REQUEST_TIMEOUT
-//  * 504 GATEWAY_TIMEOUT
-//  * 500 Internal server error
+//   - 408 REQUEST_TIMEOUT
+//   - 504 GATEWAY_TIMEOUT
+//   - 500 Internal server error
 //
 // At the end of large uploads.  The speculation is that the timeout
 // is waiting for the sha1 hashing to complete and the file may well
@@ -626,7 +626,7 @@ func (f *Fs) checkUpload(ctx context.Context, resp *http.Response, in io.Reader,
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -685,9 +685,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1002,7 +1002,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {

backend/azureblob/azureblob.go

@@ -1,8 +1,7 @@
-// Package azureblob provides an interface to the Microsoft Azure blob object storage system
-
 //go:build !plan9 && !solaris && !js
 // +build !plan9,!solaris,!js
 
+// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
 
 import (
@@ -13,9 +12,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -26,6 +25,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
@@ -43,8 +43,9 @@ import (
 const (
 	minSleep              = 10 * time.Millisecond
 	maxSleep              = 10 * time.Second
-	decayConstant         = 1    // bigger for slower decay, exponential
-	maxListChunkSize      = 5000 // number of items to read at once
+	decayConstant         = 1     // bigger for slower decay, exponential
+	maxListChunkSize      = 5000  // number of items to read at once
+	maxUploadParts        = 50000 // maximum allowed number of parts/blocks in a multi-part upload
 	modTimeKey            = "mtime"
 	timeFormatIn          = time.RFC3339
 	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
@@ -371,15 +372,9 @@ func (o *Object) split() (container, containerPath string) {
 
 // validateAccessTier checks if azureblob supports user supplied tier
 func validateAccessTier(tier string) bool {
-	switch tier {
-	case string(azblob.AccessTierHot),
-		string(azblob.AccessTierCool),
-		string(azblob.AccessTierArchive):
-		// valid cases
-		return true
-	default:
-		return false
-	}
+	return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
+		strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
+		strings.EqualFold(tier, string(azblob.AccessTierArchive))
 }
 
 // validatePublicAccess checks if azureblob supports use supplied public access level
@@ -543,10 +538,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	err = checkUploadChunkSize(opt.ChunkSize)
 	if err != nil {
-		return nil, fmt.Errorf("azure: chunk size: %w", err)
+		return nil, fmt.Errorf("chunk size: %w", err)
 	}
 	if opt.ListChunkSize > maxListChunkSize {
-		return nil, fmt.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
+		return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
 	}
 	if opt.Endpoint == "" {
 		opt.Endpoint = storageDefaultBaseURL
@@ -555,12 +550,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.AccessTier == "" {
 		opt.AccessTier = string(defaultAccessTier)
 	} else if !validateAccessTier(opt.AccessTier) {
-		return nil, fmt.Errorf("Azure Blob: Supported access tiers are %s, %s and %s",
+		return nil, fmt.Errorf("supported access tiers are %s, %s and %s",
 			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
 	}
 
 	if !validatePublicAccess((opt.PublicAccess)) {
-		return nil, fmt.Errorf("Azure Blob: Supported public access level are %s and %s",
+		return nil, fmt.Errorf("supported public access level are %s and %s",
 			string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer))
 	}
 
@@ -600,11 +595,23 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	)
 	switch {
 	case opt.UseEmulator:
-		credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey)
-		if err != nil {
-			return nil, fmt.Errorf("Failed to parse credentials: %w", err)
+		var actualEmulatorAccount = emulatorAccount
+		if opt.Account != "" {
+			actualEmulatorAccount = opt.Account
 		}
-		u, err = url.Parse(emulatorBlobEndpoint)
+		var actualEmulatorKey = emulatorAccountKey
+		if opt.Key != "" {
+			actualEmulatorKey = opt.Key
+		}
+		credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse credentials: %w", err)
+		}
+		var actualEmulatorEndpoint = emulatorBlobEndpoint
+		if opt.Endpoint != "" {
+			actualEmulatorEndpoint = opt.Endpoint
+		}
+		u, err = url.Parse(actualEmulatorEndpoint)
 		if err != nil {
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
@@ -648,7 +655,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		})
 
 		if err != nil {
-			return nil, fmt.Errorf("Failed to acquire MSI token: %w", err)
+			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
 
 		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
@@ -683,7 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	case opt.Account != "" && opt.Key != "":
 		credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
 		if err != nil {
-			return nil, fmt.Errorf("Failed to parse credentials: %w", err)
+			return nil, fmt.Errorf("failed to parse credentials: %w", err)
 		}
 
 		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
@@ -703,7 +710,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		parts := azblob.NewBlobURLParts(*u)
 		if parts.ContainerName != "" {
 			if f.rootContainer != "" && parts.ContainerName != f.rootContainer {
-				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
+				return nil, errors.New("container name in SAS URL and container provided in command do not match")
 			}
 			containerURL := azblob.NewContainerURL(*u, pipeline)
 			f.cntURLcache[parts.ContainerName] = &containerURL
@@ -718,7 +725,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
 		// Try loading service principal credentials from file.
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
 		}
@@ -731,7 +738,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options)
 		serviceURL = azblob.NewServiceURL(*u, pipe)
 	default:
-		return nil, errors.New("No authentication method configured")
+		return nil, errors.New("no authentication method configured")
 	}
 	f.svcURL = &serviceURL
 
@@ -1115,7 +1122,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1247,9 +1254,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1297,19 +1304,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
-func (f *Fs) getMemoryPool(size int64) *pool.Pool {
-	if size == int64(f.opt.ChunkSize) {
-		return f.pool
-	}
-
-	return pool.New(
-		time.Duration(f.opt.MemoryPoolFlushTime),
-		int(size),
-		f.ci.Transfers,
-		f.opt.MemoryPoolUseMmap,
-	)
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1341,7 +1335,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	data, err := base64.StdEncoding.DecodeString(o.md5)
 	if err != nil {
-		return "", fmt.Errorf("Failed to decode Content-MD5: %q: %w", o.md5, err)
+		return "", fmt.Errorf("failed to decode Content-MD5: %q: %w", o.md5, err)
 	}
 	return hex.EncodeToString(data), nil
 }
@@ -1369,11 +1363,12 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.md5
-//  o.meta
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.md5
+//	o.meta
 func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
 	metadata := info.NewMetadata()
 	size := info.ContentLength()
@@ -1456,10 +1451,11 @@ func (o *Object) clearMetaData() {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.md5
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.md5
 func (o *Object) readMetaData() (err error) {
 	container, _ := o.split()
 	if !o.fs.containerOK(container) {
@@ -1531,7 +1527,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var offset int64
 	var count int64
 	if o.AccessTier() == azblob.AccessTierArchive {
-		return nil, fmt.Errorf("Blob in archive tier, you need to set tier to hot or cool first")
+		return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot or cool first")
 	}
 	fs.FixRangeOption(options, o.size)
 	for _, option := range options {
@@ -1689,8 +1685,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
+	// Apply upload options (also allows one to overwrite content-type)
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			httpHeaders.CacheControl = value
+		case "content-disposition":
+			httpHeaders.ContentDisposition = value
+		case "content-encoding":
+			httpHeaders.ContentEncoding = value
+		case "content-language":
+			httpHeaders.ContentLanguage = value
+		case "content-type":
+			httpHeaders.ContentType = value
+		}
+	}
+
+	uploadParts := maxUploadParts
+	if uploadParts < 1 {
+		uploadParts = 1
+	} else if uploadParts > maxUploadParts {
+		uploadParts = maxUploadParts
+	}
+	// calculate size of parts/blocks
+	partSize := chunksize.Calculator(o, src.Size(), uploadParts, o.fs.opt.ChunkSize)
+
 	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize:      int(o.fs.opt.ChunkSize),
+		BufferSize:      int(partSize),
 		MaxBuffers:      o.fs.opt.UploadConcurrency,
 		Metadata:        o.meta,
 		BlobHTTPHeaders: httpHeaders,
@@ -1747,7 +1772,7 @@ func (o *Object) AccessTier() azblob.AccessTierType {
 // SetTier performs changing object tier
 func (o *Object) SetTier(tier string) error {
 	if !validateAccessTier(tier) {
-		return fmt.Errorf("Tier %s not supported by Azure Blob Storage", tier)
+		return fmt.Errorf("tier %s not supported by Azure Blob Storage", tier)
 	}
 
 	// Check if current tier already matches with desired tier
@@ -1758,12 +1783,12 @@ func (o *Object) SetTier(tier string) error {
 	blob := o.getBlobReference()
 	ctx := context.Background()
 	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
+		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}, azblob.RehydratePriorityStandard)
 		return o.fs.shouldRetry(ctx, err)
 	})
 
 	if err != nil {
-		return fmt.Errorf("Failed to set Blob Tier: %w", err)
+		return fmt.Errorf("failed to set Blob Tier: %w", err)
 	}
 
 	// Set access tier on local object also, this typically

backend/azureblob/azureblob_test.go

@@ -61,3 +61,25 @@ func TestServicePrincipalFileFailure(t *testing.T) {
 	assert.Error(t, err)
 	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
 }
+
+func TestValidateAccessTier(t *testing.T) {
+	tests := map[string]struct {
+		accessTier string
+		want       bool
+	}{
+		"hot":     {"hot", true},
+		"HOT":     {"HOT", true},
+		"Hot":     {"Hot", true},
+		"cool":    {"cool", true},
+		"archive": {"archive", true},
+		"empty":   {"", false},
+		"unknown": {"unknown", false},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			got := validateAccessTier(test.accessTier)
+			assert.Equal(t, test.want, got)
+		})
+	}
+}

backend/azureblob/imds.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/Azure/go-autorest/autorest/adal"
@@ -97,7 +96,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
 	}
 	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
+		_, err = io.Copy(io.Discard, resp.Body)
 		if err != nil {
 			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
 		}
@@ -112,14 +111,14 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 	case 200, 201, 202:
 		break
 	default:
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
 		return result, httpError{Response: resp}
 	}
 
-	b, err := ioutil.ReadAll(resp.Body)
+	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return result, fmt.Errorf("Couldn't read IMDS response: %w", err)
+		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
 	}
 	// Remove BOM, if any. azcopy does this so I'm following along.
 	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
@@ -130,7 +129,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 	// storage API call.
 	err = json.Unmarshal(b, &result)
 	if err != nil {
-		return result, fmt.Errorf("Couldn't unmarshal IMDS response: %w", err)
+		return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err)
 	}
 
 	return result, nil

backend/b2/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Backblaze B2 API.
 package api
 
 import (
@@ -238,7 +239,7 @@ type GetFileInfoRequest struct {
 // If the original source of the file being uploaded has a last
 // modified time concept, Backblaze recommends using
 // src_last_modified_millis as the name, and a string holding the base
-// 10 number number of milliseconds since midnight, January 1, 1970
+// 10 number of milliseconds since midnight, January 1, 1970
 // UTC. This fits in a 64 bit integer such as the type "long" in the
 // programming language Java. It is intended to be compatible with
 // Java's time long. For example, it can be passed directly into the

backend/b2/b2.go

@@ -1,4 +1,4 @@
-// Package b2 provides an interface to the Backblaze B2 object storage system
+// Package b2 provides an interface to the Backblaze B2 object storage system.
 package b2
 
 // FIXME should we remove sha1 checks from here as rclone now supports
@@ -64,7 +64,8 @@ const (
 
 // Globals
 var (
-	errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersions  = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
 )
 
 // Register with Fs
@@ -106,6 +107,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 			Help:     "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "version_at",
+			Help:     "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "hard_delete",
 			Help:    "Permanently delete files on remote removal, otherwise hide files.",
@@ -211,6 +217,7 @@ type Options struct {
 	Endpoint                      string               `config:"endpoint"`
 	TestMode                      string               `config:"test_mode"`
 	Versions                      bool                 `config:"versions"`
+	VersionAt                     fs.Time              `config:"version_at"`
 	HardDelete                    bool                 `config:"hard_delete"`
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
@@ -273,7 +280,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootBucket == "" {
-		return fmt.Sprintf("B2 root")
+		return "B2 root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("B2 bucket %s", f.rootBucket)
@@ -649,15 +656,15 @@ var errEndList = errors.New("end list")
 //
 // (bucket, directory) is the starting directory
 //
-// If prefix is set then it is removed from all file names
+// If prefix is set then it is removed from all file names.
 //
 // If addBucket is set then it adds the bucket to the start of the
-// remotes generated
+// remotes generated.
 //
-// If recurse is set the function will recursively list
+// If recurse is set the function will recursively list.
 //
 // If limit is > 0 then it limits to that many files (must be less
-// than 1000)
+// than 1000).
 //
 // If hidden is set then it will list the hidden (deleted) files too.
 //
@@ -696,9 +703,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		Method: "POST",
 		Path:   "/b2_list_file_names",
 	}
-	if hidden {
+	if hidden || f.opt.VersionAt.IsSet() {
 		opts.Path = "/b2_list_file_versions"
 	}
+
+	lastFileName := ""
+
 	for {
 		var response api.ListFileNamesResponse
 		err := f.pacer.Call(func() (bool, error) {
@@ -728,7 +738,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
+
+			if f.opt.VersionAt.IsSet() {
+				if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
+					// Ignore versions that were created after the specified time
+					continue
+				}
+
+				if file.Name == lastFileName {
+					// Ignore versions before the already returned version
+					continue
+				}
+			}
+
 			// Send object
+			lastFileName = file.Name
 			err = fn(remote, file, isDirectory)
 			if err != nil {
 				if err == errEndList {
@@ -1001,7 +1025,7 @@ func (f *Fs) clearBucketID(bucket string) {
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1181,10 +1205,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 		}
 	}
 	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		if time.Since(time.Time(timestamp)).Hours() > 24 {
-			return true
-		}
-		return false
+		return time.Since(time.Time(timestamp)).Hours() > 24
 	}
 
 	// Delete Config.Transfers in parallel
@@ -1313,9 +1334,9 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1457,26 +1478,23 @@ func (o *Object) Size() int64 {
 
 // Clean the SHA1
 //
-// Make sure it is lower case
+// Make sure it is lower case.
 //
 // Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
 // Some tools (e.g. Cyberduck) use this
-func cleanSHA1(sha1 string) (out string) {
-	out = strings.ToLower(sha1)
+func cleanSHA1(sha1 string) string {
 	const unverified = "unverified:"
-	if strings.HasPrefix(out, unverified) {
-		out = out[len(unverified):]
-	}
-	return out
+	return strings.TrimPrefix(strings.ToLower(sha1), unverified)
 }
 
 // decodeMetaDataRaw sets the metadata from the data passed in
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) {
 	o.id = ID
 	o.sha1 = SHA1
@@ -1495,10 +1513,11 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 // decodeMetaData sets the metadata in the object from an api.File
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaData(info *api.File) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1506,10 +1525,11 @@ func (o *Object) decodeMetaData(info *api.File) (err error) {
 // decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1567,10 +1587,11 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.id != "" {
 		return nil
@@ -1828,6 +1849,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	size := src.Size()
 
 	bucket, bucketPath := o.split()
@@ -1983,6 +2007,9 @@ func (o *Object) Remove(ctx context.Context) error {
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	if o.fs.opt.HardDelete {
 		return o.fs.deleteByID(ctx, o.id, bucketPath)
 	}

backend/b2/upload.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
@@ -88,21 +89,19 @@ type largeUpload struct {
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
-	remote := o.remote
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
 	parts := int64(0)
 	sha1SliceSize := int64(maxParts)
+	chunkSize := defaultChunkSize
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
+		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
 		parts = size / int64(chunkSize)
 		if size%int64(chunkSize) != 0 {
 			parts++
 		}
-		if parts > maxParts {
-			return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
-		}
 		sha1SliceSize = parts
 	}
 

backend/box/api/types.go

@@ -14,7 +14,7 @@ const (
 	timeFormat = `"` + time.RFC3339 + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // box API, by using RFC3339
 type Time time.Time
 

backend/box/box.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -183,7 +183,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := ioutil.ReadFile(configFile)
+	file, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}
@@ -266,7 +266,7 @@ type Fs struct {
 	root         string                // the path we are working on
 	opt          Options               // parsed options
 	features     *fs.Features          // optional features
-	srv          *rest.Client          // the connection to the one drive server
+	srv          *rest.Client          // the connection to the server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
 	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
@@ -692,7 +692,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -752,7 +752,7 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -792,9 +792,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // PutUnchecked the object into the container
 //
-// This will produce an error if the object already exists
+// This will produce an error if the object already exists.
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -877,9 +877,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -897,7 +897,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	srcPath := srcObj.fs.rootSlash() + srcObj.remote
 	dstPath := f.rootSlash() + remote
-	if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
+	if strings.EqualFold(srcPath, dstPath) {
 		return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
 	}
 
@@ -995,9 +995,9 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1235,7 +1235,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1346,9 +1345,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
-// The new object may have been created if an error is returned
+// The new object may have been created if an error is returned.
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	if o.fs.tokenRenewer != nil {
 		o.fs.tokenRenewer.Start()

backend/cache/cache.go

@@ -1,6 +1,7 @@
 //go:build !plan9 && !js
 // +build !plan9,!js
 
+// Package cache implements a virtual provider to cache existing remotes.
 package cache
 
 import (
@@ -1128,7 +1129,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				case fs.Directory:
 					_ = f.cache.AddDir(DirectoryFromOriginal(ctx, f, o))
 				default:
-					return fmt.Errorf("Unknown object type %T", entry)
+					return fmt.Errorf("unknown object type %T", entry)
 				}
 			}
 
@@ -1747,7 +1748,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }

backend/cache/cache_internal_test.go

@@ -11,7 +11,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -167,7 +166,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -841,7 +840,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
+		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -984,7 +983,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := ioutil.TempFile("", "rclonecache-tempfile")
+	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {

backend/cache/cache_test.go

@@ -19,7 +19,7 @@ func TestIntegration(t *testing.T) {
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
 		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
 		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/plex.go

@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = ioutil.ReadAll(resp.Body)
+							data, err = io.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}
@@ -213,7 +213,7 @@ func (p *plexConnector) authenticate() error {
 	var data map[string]interface{}
 	err = json.NewDecoder(resp.Body).Decode(&data)
 	if err != nil {
-		return fmt.Errorf("failed to obtain token: %v", err)
+		return fmt.Errorf("failed to obtain token: %w", err)
 	}
 	tokenGen, ok := get(data, "user", "authToken")
 	if !ok {

backend/cache/storage_memory.go

@@ -76,10 +76,7 @@ func (m *Memory) CleanChunksByAge(chunkAge time.Duration) {
 
 // CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk
 func (m *Memory) CleanChunksByNeed(offset int64) {
-	var items map[string]cache.Item
-
-	items = m.db.Items()
-	for key := range items {
+	for key := range m.db.Items() {
 		sepIdx := strings.LastIndex(key, "-")
 		keyOffset, err := strconv.ParseInt(key[sepIdx+1:], 10, 64)
 		if err != nil {

backend/cache/storage_persistent.go

@@ -9,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -250,7 +249,7 @@ func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)
 		if val != nil {
 			err := json.Unmarshal(val, cachedDir)
 			if err != nil {
-				return fmt.Errorf("error during unmarshalling obj: %v", err)
+				return fmt.Errorf("error during unmarshalling obj: %w", err)
 			}
 		} else {
 			return fmt.Errorf("missing cached dir: %v", cachedDir)
@@ -456,10 +455,7 @@ func (b *Persistent) HasEntry(remote string) bool {
 
 		return fmt.Errorf("couldn't find object (%v)", remote)
 	})
-	if err == nil {
-		return true
-	}
-	return false
+	return err == nil
 }
 
 // HasChunk confirms the existence of a single chunk of an object
@@ -476,7 +472,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := ioutil.ReadFile(fp)
+	data, err := os.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -489,7 +485,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := ioutil.WriteFile(filePath, data, os.ModePerm)
+	err := os.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}
@@ -554,7 +550,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		dataTsBucket := tx.Bucket([]byte(DataTsBucket))
 		if dataTsBucket == nil {
-			return fmt.Errorf("Couldn't open (%v) bucket", DataTsBucket)
+			return fmt.Errorf("couldn't open (%v) bucket", DataTsBucket)
 		}
 		// iterate through ts
 		c := dataTsBucket.Cursor()
@@ -904,16 +900,16 @@ func (b *Persistent) rollbackPendingUpload(remote string) error {
 		v := bucket.Get([]byte(remote))
 		err = json.Unmarshal(v, tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload (%v) not found %v", remote, err)
+			return fmt.Errorf("pending upload (%v) not found: %w", remote, err)
 		}
 		tempObj.Started = false
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		return nil
 	})
@@ -969,11 +965,11 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 		}
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated %v", err)
+			return fmt.Errorf("pending upload not updated: %w", err)
 		}
 
 		return nil

backend/chunker/chunker.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -32,7 +31,6 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 )
 
-//
 // Chunker's composite files have one or more chunks
 // and optional metadata object. If it's present,
 // meta object is named after the original file.
@@ -65,7 +63,7 @@ import (
 // length of 13 decimals it makes a 7-digit base-36 number.
 //
 // When transactions is set to the norename style, data chunks will
-// keep their temporary chunk names (with the transacion identifier
+// keep their temporary chunk names (with the transaction identifier
 // suffix). To distinguish them from temporary chunks, the txn field
 // of the metadata file is set to match the transaction identifier of
 // the data chunks.
@@ -79,7 +77,6 @@ import (
 // Metadata format v1 does not define any control chunk types,
 // they are currently ignored aka reserved.
 // In future they can be used to implement resumable uploads etc.
-//
 const (
 	ctrlTypeRegStr   = `[a-z][a-z0-9]{2,6}`
 	tempSuffixFormat = `_%04s`
@@ -515,7 +512,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 
 	strRegex := regexp.QuoteMeta(pattern)
 	strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
-	strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
+	strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
 	strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
 	f.nameRegexp = regexp.MustCompile(strRegex)
 
@@ -524,7 +521,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 	if numDigits > 1 {
 		fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
 	}
-	strFmt := strings.Replace(pattern, "%", "%%", -1)
+	strFmt := strings.ReplaceAll(pattern, "%", "%%")
 	strFmt = strings.Replace(strFmt, "*", "%s", 1)
 	f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
 	f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
@@ -542,7 +539,6 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 //
 // xactID is a transaction identifier. Empty xactID denotes active chunk,
 // otherwise temporary chunk name is produced.
-//
 func (f *Fs) makeChunkName(filePath string, chunkNo int, ctrlType, xactID string) string {
 	dir, parentName := path.Split(filePath)
 	var name, tempSuffix string
@@ -708,7 +704,6 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
 // directory together with dead chunks.
 // In future a flag named like `--chunker-list-hidden` may be added to
 // rclone that will tell List to reveal hidden chunks.
-//
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	entries, err = f.base.List(ctx, dir)
 	if err != nil {
@@ -868,7 +863,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 // Note that chunker prefers analyzing file names rather than reading
 // the content of meta object assuming that directory scans are fast
 // but opening even a small file can be slow on some backends.
-//
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.scanObject(ctx, remote, false)
 }
@@ -1043,7 +1037,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := ioutil.ReadAll(reader)
+	metadata, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1084,7 +1078,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 
 // readXactID returns the transaction ID stored in the passed metadata object
 func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
-	// if xactID has already been read and cahced return it now
+	// if xactID has already been read and cached return it now
 	if o.xIDCached {
 		return o.xactID, nil
 	}
@@ -1102,7 +1096,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadAll(reader)
+	data, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err
@@ -1586,7 +1580,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 // This command will chain to `purge` from wrapped remote.
 // As a result it removes not only composite chunker files with their
 // active chunks but also all hidden temporary chunks in the directory.
-//
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	do := f.base.Features().Purge
 	if do == nil {
@@ -1628,7 +1621,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Unsupported control chunks will get re-picked by a more recent
 // rclone version with unexpected results. This can be helped by
 // the `delete hidden` flag above or at least the user has been warned.
-//
 func (o *Object) Remove(ctx context.Context) (err error) {
 	if err := o.f.forbidChunk(o, o.Remote()); err != nil {
 		// operations.Move can still call Remove if chunker's Move refuses
@@ -1804,9 +1796,9 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string)
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1825,9 +1817,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1895,7 +1887,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.base.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1904,7 +1896,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.base.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -2125,7 +2117,6 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
 // file, then tries to read it from metadata. This in theory
 // handles the unusual case when a small file has been tampered
 // on the level of wrapped remote but chunker is unaware of that.
-//
 func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) {
 	if err := o.readMetadata(ctx); err != nil {
 		return "", err // valid metadata is required to get hash, abort
@@ -2414,7 +2405,6 @@ type metaSimpleJSON struct {
 // - for files larger than chunk size
 // - if file contents can be mistaken as meta object
 // - if consistent hashing is On but wrapped remote can't provide given hash
-//
 func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) {
 	version := metadataVersion
 	if xactID == "" && version == 2 {
@@ -2447,7 +2437,6 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1,
 // New format will have a higher version number and cannot be correctly
 // handled by current implementation.
 // The version check below will then explicitly ask user to upgrade rclone.
-//
 func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) {
 	// Be strict about JSON format
 	// to reduce possibility that a random small file resembles metadata.

backend/chunker/chunker_internal_test.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -59,7 +59,7 @@ var mtime1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
 
 func testPutFile(ctx context.Context, t *testing.T, f fs.Fs, name, contents, message string, check bool) fs.Object {
 	item := fstest.Item{Path: name, ModTime: mtime1}
-	_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
+	obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
 	assert.NotNil(t, obj, message)
 	return obj
 }
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -440,7 +440,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 	checkSmallFile := func(name, contents string) {
 		filename := path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
+		put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
 		assert.NotNil(t, put)
 		checkSmallFileInternals(put)
 		checkContents(put, contents)
@@ -489,7 +489,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 
 	newFile := func(name string) fs.Object {
 		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		return obj
 	}
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
+		chunkContents, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
+		_, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -599,7 +599,7 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
 		filename = path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
 			txnID = chunkObj.xactID
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
+			data, err := io.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -716,7 +716,7 @@ func testFutureProof(t *testing.T, f *Fs) {
 			name = f.makeChunkName(name, part-1, "", "")
 		}
 		item := fstest.Item{Path: name, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
+		obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
 		assert.NotNil(t, obj, msg)
 	}
 
@@ -758,8 +758,8 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
+	in := io.NopCloser(bytes.NewBufferString("abc"))
+	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
@@ -790,7 +790,7 @@ func testBackwardsCompatibility(t *testing.T, f *Fs) {
 	newFile := func(f fs.Fs, name string) (fs.Object, string) {
 		filename := path.Join(dir, name)
 		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
 		require.NotNil(t, obj)
 		return obj, filename
 	}
@@ -844,7 +844,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	item := fstest.Item{Path: "movefile", ModTime: modTime}
 	contents := "abcdef"
-	_, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
+	file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
 
 	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
 	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
+	data, err := io.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()

backend/chunker/chunker_test.go

@@ -35,6 +35,7 @@ func TestIntegration(t *testing.T) {
 			"MimeType",
 			"GetTier",
 			"SetTier",
+			"Metadata",
 		},
 		UnimplementableFsMethods: []string{
 			"PublicLink",
@@ -53,6 +54,7 @@ func TestIntegration(t *testing.T) {
 			{Name: name, Key: "type", Value: "chunker"},
 			{Name: name, Key: "remote", Value: tempDir},
 		}
+		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
 }

backend/combine/combine.go

@@ -0,0 +1,992 @@
+// Package combine implents a backend to combine multiple remotes in a directory tree
+package combine
+
+/*
+   Have API to add/remove branches in the combine
+*/
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/walk"
+	"golang.org/x/sync/errgroup"
+)
+
+// Register with Fs
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "combine",
+		Description: "Combine several remotes into one",
+		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
+		Options: []fs.Option{{
+			Name: "upstreams",
+			Help: `Upstreams for combining
+
+These should be in the form
+
+    dir=remote:path dir2=remote2:path
+
+Where before the = is specified the root directory and after is the remote to
+put there.
+
+Embedded spaces can be added using quotes
+
+    "dir=remote:path with space" "dir2=remote2:path with space"
+
+`,
+			Required: true,
+			Default:  fs.SpaceSepList(nil),
+		}},
+	}
+	fs.Register(fsi)
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Upstreams fs.SpaceSepList `config:"upstreams"`
+}
+
+// Fs represents a combine of upstreams
+type Fs struct {
+	name      string               // name of this remote
+	features  *fs.Features         // optional features
+	opt       Options              // options for this Fs
+	root      string               // the path we are working on
+	hashSet   hash.Set             // common hashes
+	when      time.Time            // directory times
+	upstreams map[string]*upstream // map of upstreams
+}
+
+// adjustment stores the info to add a prefix to a path or chop characters off
+type adjustment struct {
+	root            string
+	rootSlash       string
+	mountpoint      string
+	mountpointSlash string
+}
+
+// newAdjustment makes a new path adjustment adjusting between mountpoint and root
+//
+// mountpoint is the point the upstream is mounted and root is the combine root
+func newAdjustment(root, mountpoint string) (a adjustment) {
+	return adjustment{
+		root:            root,
+		rootSlash:       root + "/",
+		mountpoint:      mountpoint,
+		mountpointSlash: mountpoint + "/",
+	}
+}
+
+var errNotUnderRoot = errors.New("file not under root")
+
+// do makes the adjustment on s, mapping an upstream path into a combine path
+func (a *adjustment) do(s string) (string, error) {
+	absPath := join(a.mountpoint, s)
+	if a.root == "" {
+		return absPath, nil
+	}
+	if absPath == a.root {
+		return "", nil
+	}
+	if !strings.HasPrefix(absPath, a.rootSlash) {
+		return "", errNotUnderRoot
+	}
+	return absPath[len(a.rootSlash):], nil
+}
+
+// undo makes the adjustment on s, mapping a combine path into an upstream path
+func (a *adjustment) undo(s string) (string, error) {
+	absPath := join(a.root, s)
+	if absPath == a.mountpoint {
+		return "", nil
+	}
+	if !strings.HasPrefix(absPath, a.mountpointSlash) {
+		return "", errNotUnderRoot
+	}
+	return absPath[len(a.mountpointSlash):], nil
+}
+
+// upstream represents an upstream Fs
+type upstream struct {
+	f              fs.Fs
+	parent         *Fs
+	dir            string     // directory the upstream is mounted
+	pathAdjustment adjustment // how to fiddle with the path
+}
+
+// Create an upstream from the directory it is mounted on and the remote
+func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, error) {
+	uFs, err := cache.Get(ctx, remote)
+	if err == fs.ErrorIsFile {
+		return nil, fmt.Errorf("can't combine files yet, only directories %q: %w", remote, err)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to create upstream %q: %w", remote, err)
+	}
+	u := &upstream{
+		f:              uFs,
+		parent:         f,
+		dir:            dir,
+		pathAdjustment: newAdjustment(f.root, dir),
+	}
+	cache.PinUntilFinalized(u.f, u)
+	return u, nil
+}
+
+// NewFs constructs an Fs from the path.
+//
+// The returned Fs is the actual Fs, referenced by remote in the config
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
+	// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
+	// Parse config into Options struct
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	// Backward compatible to old config
+	if len(opt.Upstreams) == 0 {
+		return nil, errors.New("combine can't point to an empty upstream - check the value of the upstreams setting")
+	}
+	for _, u := range opt.Upstreams {
+		if strings.HasPrefix(u, name+":") {
+			return nil, errors.New("can't point combine remote at itself - check the value of the upstreams setting")
+		}
+	}
+	isDir := false
+	for strings.HasSuffix(root, "/") {
+		root = root[:len(root)-1]
+		isDir = true
+	}
+
+	f := &Fs{
+		name:      name,
+		root:      root,
+		opt:       *opt,
+		upstreams: make(map[string]*upstream, len(opt.Upstreams)),
+		when:      time.Now(),
+	}
+
+	g, gCtx := errgroup.WithContext(ctx)
+	var mu sync.Mutex
+	for _, upstream := range opt.Upstreams {
+		upstream := upstream
+		g.Go(func() (err error) {
+			equal := strings.IndexRune(upstream, '=')
+			if equal < 0 {
+				return fmt.Errorf("no \"=\" in upstream definition %q", upstream)
+			}
+			dir, remote := upstream[:equal], upstream[equal+1:]
+			if dir == "" {
+				return fmt.Errorf("empty dir in upstream definition %q", upstream)
+			}
+			if remote == "" {
+				return fmt.Errorf("empty remote in upstream definition %q", upstream)
+			}
+			if strings.ContainsRune(dir, '/') {
+				return fmt.Errorf("dirs can't contain / (yet): %q", dir)
+			}
+			u, err := f.newUpstream(gCtx, dir, remote)
+			if err != nil {
+				return err
+			}
+			mu.Lock()
+			if _, found := f.upstreams[dir]; found {
+				err = fmt.Errorf("duplicate directory name %q", dir)
+			} else {
+				f.upstreams[dir] = u
+			}
+			mu.Unlock()
+			return err
+		})
+	}
+	err = g.Wait()
+	if err != nil {
+		return nil, err
+	}
+	// check features
+	var features = (&fs.Features{
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		BucketBased:             true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+	}).Fill(ctx, f)
+	canMove := true
+	for _, u := range f.upstreams {
+		features = features.Mask(ctx, u.f) // Mask all upstream fs
+		if !operations.CanServerSideMove(u.f) {
+			canMove = false
+		}
+	}
+	// We can move if all remotes support Move or Copy
+	if canMove {
+		features.Move = f.Move
+	}
+
+	// Enable ListR when upstreams either support ListR or is local
+	// But not when all upstreams are local
+	if features.ListR == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().ListR != nil {
+				features.ListR = f.ListR
+			} else if !u.f.Features().IsLocal {
+				features.ListR = nil
+				break
+			}
+		}
+	}
+
+	// Enable Purge when any upstreams support it
+	if features.Purge == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().Purge != nil {
+				features.Purge = f.Purge
+				break
+			}
+		}
+	}
+
+	// Enable Shutdown when any upstreams support it
+	if features.Shutdown == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().Shutdown != nil {
+				features.Shutdown = f.Shutdown
+				break
+			}
+		}
+	}
+
+	// Enable DirCacheFlush when any upstreams support it
+	if features.DirCacheFlush == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().DirCacheFlush != nil {
+				features.DirCacheFlush = f.DirCacheFlush
+				break
+			}
+		}
+	}
+
+	// Enable ChangeNotify when any upstreams support it
+	if features.ChangeNotify == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().ChangeNotify != nil {
+				features.ChangeNotify = f.ChangeNotify
+				break
+			}
+		}
+	}
+
+	f.features = features
+
+	// Get common intersection of hashes
+	var hashSet hash.Set
+	var first = true
+	for _, u := range f.upstreams {
+		if first {
+			hashSet = u.f.Hashes()
+			first = false
+		} else {
+			hashSet = hashSet.Overlap(u.f.Hashes())
+		}
+	}
+	f.hashSet = hashSet
+
+	// Check to see if the root is actually a file
+	if f.root != "" && !isDir {
+		_, err := f.NewObject(ctx, "")
+		if err != nil {
+			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile || err == fs.ErrorIsDir {
+				// File doesn't exist or is a directory so return old f
+				return f, nil
+			}
+			return nil, err
+		}
+
+		// Check to see if the root path is actually an existing file
+		f.root = path.Dir(f.root)
+		if f.root == "." {
+			f.root = ""
+		}
+		// Adjust path adjustment to remove leaf
+		for _, u := range f.upstreams {
+			u.pathAdjustment = newAdjustment(f.root, u.dir)
+		}
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// Run a function over all the upstreams in parallel
+func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error {
+	g, gCtx := errgroup.WithContext(ctx)
+	for _, u := range f.upstreams {
+		u := u
+		g.Go(func() (err error) {
+			return fn(gCtx, u)
+		})
+	}
+	return g.Wait()
+}
+
+// join the elements together but unline path.Join return empty string
+func join(elem ...string) string {
+	result := path.Join(elem...)
+	if result == "." {
+		return ""
+	}
+	if len(result) > 0 && result[0] == '/' {
+		result = result[1:]
+	}
+	return result
+}
+
+// find the upstream for the remote passed in, returning the upstream and the adjusted path
+func (f *Fs) findUpstream(remote string) (u *upstream, uRemote string, err error) {
+	// defer log.Trace(remote, "")("f=%v, uRemote=%q, err=%v", &u, &uRemote, &err)
+	for _, u := range f.upstreams {
+		uRemote, err = u.pathAdjustment.undo(remote)
+		if err == nil {
+			return u, uRemote, nil
+		}
+	}
+	return nil, "", fmt.Errorf("combine for remote %q: %w", remote, fs.ErrorDirNotFound)
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("combine root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Rmdir removes the root directory of the Fs object
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	// The root always exists
+	if f.root == "" && dir == "" {
+		return nil
+	}
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return err
+	}
+	return u.f.Rmdir(ctx, uRemote)
+}
+
+// Hashes returns hash.HashNone to indicate remote hashing is unavailable
+func (f *Fs) Hashes() hash.Set {
+	return f.hashSet
+}
+
+// Mkdir makes the root directory of the Fs object
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	// The root always exists
+	if f.root == "" && dir == "" {
+		return nil
+	}
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return err
+	}
+	return u.f.Mkdir(ctx, uRemote)
+}
+
+// purge the upstream or fallback to a slow way
+func (u *upstream) purge(ctx context.Context, dir string) (err error) {
+	if do := u.f.Features().Purge; do != nil {
+		err = do(ctx, dir)
+	} else {
+		err = operations.Purge(ctx, u.f, dir)
+	}
+	return err
+}
+
+// Purge all files in the directory
+//
+// Implement this if you have a way of deleting all the files
+// quicker than just running Remove() on the result of List()
+//
+// Return an error if it doesn't exist
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	if f.root == "" && dir == "" {
+		return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
+			return u.purge(ctx, "")
+		})
+	}
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return err
+	}
+	return u.purge(ctx, uRemote)
+}
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	dstU, dstRemote, err := f.findUpstream(remote)
+	if err != nil {
+		return nil, err
+	}
+
+	do := dstU.f.Features().Copy
+	if do == nil {
+		return nil, fs.ErrorCantCopy
+	}
+
+	o, err := do(ctx, srcObj.Object, dstRemote)
+	if err != nil {
+		return nil, err
+	}
+
+	return dstU.newObject(o), nil
+}
+
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	dstU, dstRemote, err := f.findUpstream(remote)
+	if err != nil {
+		return nil, err
+	}
+
+	do := dstU.f.Features().Move
+	useCopy := false
+	if do == nil {
+		do = dstU.f.Features().Copy
+		if do == nil {
+			return nil, fs.ErrorCantMove
+		}
+		useCopy = true
+	}
+
+	o, err := do(ctx, srcObj.Object, dstRemote)
+	if err != nil {
+		return nil, err
+	}
+
+	// If did Copy then remove the source object
+	if useCopy {
+		err = srcObj.Remove(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return dstU.newObject(o), nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	// defer log.Trace(f, "src=%v, srcRemote=%q, dstRemote=%q", src, srcRemote, dstRemote)("err=%v", &err)
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(src, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	dstU, dstURemote, err := f.findUpstream(dstRemote)
+	if err != nil {
+		return err
+	}
+
+	srcU, srcURemote, err := srcFs.findUpstream(srcRemote)
+	if err != nil {
+		return err
+	}
+
+	do := dstU.f.Features().DirMove
+	if do == nil {
+		return fs.ErrorCantDirMove
+	}
+
+	fs.Logf(dstU.f, "srcU.f=%v, srcURemote=%q, dstURemote=%q", srcU.f, srcURemote, dstURemote)
+	return do(ctx, srcU.f, srcURemote, dstURemote)
+}
+
+// ChangeNotify calls the passed function with a path
+// that has had changes. If the implementation
+// uses polling, it should adhere to the given interval.
+// At least one value will be written to the channel,
+// specifying the initial value and updated values might
+// follow. A 0 Duration should pause the polling.
+// The ChangeNotify implementation must empty the channel
+// regularly. When the channel gets closed, the implementation
+// should stop polling and release resources.
+func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) {
+	var uChans []chan time.Duration
+
+	for _, u := range f.upstreams {
+		u := u
+		if do := u.f.Features().ChangeNotify; do != nil {
+			ch := make(chan time.Duration)
+			uChans = append(uChans, ch)
+			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+				newPath, err := u.pathAdjustment.do(path)
+				if err != nil {
+					fs.Logf(f, "ChangeNotify: unable to process %q: %s", path, err)
+					return
+				}
+				fs.Debugf(f, "ChangeNotify: path %q entryType %d", newPath, entryType)
+				notifyFunc(newPath, entryType)
+			}
+			do(ctx, wrappedNotifyFunc, ch)
+		}
+	}
+
+	go func() {
+		for i := range ch {
+			for _, c := range uChans {
+				c <- i
+			}
+		}
+		for _, c := range uChans {
+			close(c)
+		}
+	}()
+}
+
+// DirCacheFlush resets the directory cache - used in testing
+// as an optional interface
+func (f *Fs) DirCacheFlush() {
+	ctx := context.Background()
+	_ = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
+		if do := u.f.Features().DirCacheFlush; do != nil {
+			do()
+		}
+		return nil
+	})
+}
+
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
+	srcPath := src.Remote()
+	u, uRemote, err := f.findUpstream(srcPath)
+	if err != nil {
+		return nil, err
+	}
+	uSrc := operations.NewOverrideRemote(src, uRemote)
+	var o fs.Object
+	if stream {
+		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)
+	} else {
+		o, err = u.f.Put(ctx, in, uSrc, options...)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return u.newObject(o), nil
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return o, o.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, false, options...)
+	default:
+		return nil, err
+	}
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return o, o.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, true, options...)
+	default:
+		return nil, err
+	}
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	usage := &fs.Usage{
+		Total:   new(int64),
+		Used:    new(int64),
+		Trashed: new(int64),
+		Other:   new(int64),
+		Free:    new(int64),
+		Objects: new(int64),
+	}
+	for _, u := range f.upstreams {
+		doAbout := u.f.Features().About
+		if doAbout == nil {
+			continue
+		}
+		usg, err := doAbout(ctx)
+		if errors.Is(err, fs.ErrorDirNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		if usg.Total != nil && usage.Total != nil {
+			*usage.Total += *usg.Total
+		} else {
+			usage.Total = nil
+		}
+		if usg.Used != nil && usage.Used != nil {
+			*usage.Used += *usg.Used
+		} else {
+			usage.Used = nil
+		}
+		if usg.Trashed != nil && usage.Trashed != nil {
+			*usage.Trashed += *usg.Trashed
+		} else {
+			usage.Trashed = nil
+		}
+		if usg.Other != nil && usage.Other != nil {
+			*usage.Other += *usg.Other
+		} else {
+			usage.Other = nil
+		}
+		if usg.Free != nil && usage.Free != nil {
+			*usage.Free += *usg.Free
+		} else {
+			usage.Free = nil
+		}
+		if usg.Objects != nil && usage.Objects != nil {
+			*usage.Objects += *usg.Objects
+		} else {
+			usage.Objects = nil
+		}
+	}
+	return usage, nil
+}
+
+// Wraps entries for this upstream
+func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.DirEntries, error) {
+	for i, entry := range entries {
+		switch x := entry.(type) {
+		case fs.Object:
+			entries[i] = u.newObject(x)
+		case fs.Directory:
+			newDir := fs.NewDirCopy(ctx, x)
+			newPath, err := u.pathAdjustment.do(newDir.Remote())
+			if err != nil {
+				return nil, err
+			}
+			newDir.SetRemote(newPath)
+			entries[i] = newDir
+		default:
+			return nil, fmt.Errorf("unknown entry type %T", entry)
+		}
+	}
+	return entries, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
+	if f.root == "" && dir == "" {
+		entries = make(fs.DirEntries, 0, len(f.upstreams))
+		for combineDir := range f.upstreams {
+			d := fs.NewDir(combineDir, f.when)
+			entries = append(entries, d)
+		}
+		return entries, nil
+	}
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return nil, err
+	}
+	entries, err = u.f.List(ctx, uRemote)
+	if err != nil {
+		return nil, err
+	}
+	return u.wrapEntries(ctx, entries)
+}
+
+// ListR lists the objects and directories of the Fs starting
+// from dir recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+//
+// Don't implement this unless you have a more efficient way
+// of listing recursively that doing a directory traversal.
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	// defer log.Trace(f, "dir=%q, callback=%v", dir, callback)("err=%v", &err)
+	if f.root == "" && dir == "" {
+		rootEntries, err := f.List(ctx, "")
+		if err != nil {
+			return err
+		}
+		err = callback(rootEntries)
+		if err != nil {
+			return err
+		}
+		var mu sync.Mutex
+		syncCallback := func(entries fs.DirEntries) error {
+			mu.Lock()
+			defer mu.Unlock()
+			return callback(entries)
+		}
+		err = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
+			return f.ListR(ctx, u.dir, syncCallback)
+		})
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return err
+	}
+	wrapCallback := func(entries fs.DirEntries) error {
+		entries, err := u.wrapEntries(ctx, entries)
+		if err != nil {
+			return err
+		}
+		return callback(entries)
+	}
+	if do := u.f.Features().ListR; do != nil {
+		err = do(ctx, uRemote, wrapCallback)
+	} else {
+		err = walk.ListR(ctx, u.f, uRemote, true, -1, walk.ListAll, wrapCallback)
+	}
+	if err == fs.ErrorDirNotFound {
+		err = nil
+	}
+	return err
+}
+
+// NewObject creates a new remote combine file object
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	u, uRemote, err := f.findUpstream(remote)
+	if err != nil {
+		return nil, err
+	}
+	if uRemote == "" || strings.HasSuffix(uRemote, "/") {
+		return nil, fs.ErrorIsDir
+	}
+	o, err := u.f.NewObject(ctx, uRemote)
+	if err != nil {
+		return nil, err
+	}
+	return u.newObject(o), nil
+}
+
+// Precision is the greatest Precision of all upstreams
+func (f *Fs) Precision() time.Duration {
+	var greatestPrecision time.Duration
+	for _, u := range f.upstreams {
+		uPrecision := u.f.Precision()
+		if uPrecision > greatestPrecision {
+			greatestPrecision = uPrecision
+		}
+	}
+	return greatestPrecision
+}
+
+// Shutdown the backend, closing any background tasks and any
+// cached connections.
+func (f *Fs) Shutdown(ctx context.Context) error {
+	return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
+		if do := u.f.Features().Shutdown; do != nil {
+			return do(ctx)
+		}
+		return nil
+	})
+}
+
+// Object describes a wrapped Object
+//
+// This is a wrapped Object which knows its path prefix
+type Object struct {
+	fs.Object
+	u *upstream
+}
+
+func (u *upstream) newObject(o fs.Object) *Object {
+	return &Object{
+		Object: o,
+		u:      u,
+	}
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.u.parent
+}
+
+// String returns the remote path
+func (o *Object) String() string {
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	newPath, err := o.u.pathAdjustment.do(o.Object.String())
+	if err != nil {
+		fs.Errorf(o, "Bad object: %v", err)
+		return err.Error()
+	}
+	return newPath
+}
+
+// MimeType returns the content type of the Object if known
+func (o *Object) MimeType(ctx context.Context) (mimeType string) {
+	if do, ok := o.Object.(fs.MimeTyper); ok {
+		mimeType = do.MimeType(ctx)
+	}
+	return mimeType
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *Object) UnWrap() fs.Object {
+	return o.Object
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *Object) GetTier() string {
+	do, ok := o.Object.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	do, ok := o.Object.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.Object.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// SetTier performs changing storage tier of the Object if
+// multiple storage classes supported
+func (o *Object) SetTier(tier string) error {
+	do, ok := o.Object.(fs.SetTierer)
+	if !ok {
+		return errors.New("underlying remote does not support SetTier")
+	}
+	return do.SetTier(tier)
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Purger          = (*Fs)(nil)
+	_ fs.PutStreamer     = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.ListRer         = (*Fs)(nil)
+	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.FullObject      = (*Object)(nil)
+)

backend/combine/combine_internal_test.go

@@ -0,0 +1,94 @@
+package combine
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAdjustmentDo(t *testing.T) {
+	for _, test := range []struct {
+		root       string
+		mountpoint string
+		in         string
+		want       string
+		wantErr    error
+	}{
+		{
+			root:       "",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "mountpoint/path/to/file.txt",
+		},
+		{
+			root:       "mountpoint",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "wrongpath/to/file.txt",
+			want:       "",
+			wantErr:    errNotUnderRoot,
+		},
+	} {
+		what := fmt.Sprintf("%+v", test)
+		a := newAdjustment(test.root, test.mountpoint)
+		got, gotErr := a.do(test.in)
+		assert.Equal(t, test.wantErr, gotErr)
+		assert.Equal(t, test.want, got, what)
+	}
+
+}
+
+func TestAdjustmentUndo(t *testing.T) {
+	for _, test := range []struct {
+		root       string
+		mountpoint string
+		in         string
+		want       string
+		wantErr    error
+	}{
+		{
+			root:       "",
+			mountpoint: "mountpoint",
+			in:         "mountpoint/path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint",
+			mountpoint: "mountpoint",
+			in:         "path/to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "mountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "to/file.txt",
+			want:       "path/to/file.txt",
+		},
+		{
+			root:       "wrongmountpoint/path",
+			mountpoint: "mountpoint",
+			in:         "to/file.txt",
+			want:       "",
+			wantErr:    errNotUnderRoot,
+		},
+	} {
+		what := fmt.Sprintf("%+v", test)
+		a := newAdjustment(test.root, test.mountpoint)
+		got, gotErr := a.undo(test.in)
+		assert.Equal(t, test.wantErr, gotErr)
+		assert.Equal(t, test.want, got, what)
+	}
+
+}

backend/combine/combine_test.go

@@ -0,0 +1,81 @@
+// Test Combine filesystem interface
+package combine_test
+
+import (
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	_ "github.com/rclone/rclone/backend/memory"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	if *fstest.RemoteName == "" {
+		t.Skip("Skipping as -remote not set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:                   *fstest.RemoteName,
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
+
+func TestLocal(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	dirs := MakeTestDirs(t, 3)
+	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=" + dirs[2]
+	name := "TestCombineLocal"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+		QuickTestOK: true,
+	})
+}
+
+func TestMemory(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	upstreams := "dir1=:memory:dir1 dir2=:memory:dir2 dir3=:memory:dir3"
+	name := "TestCombineMemory"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+		QuickTestOK: true,
+	})
+}
+
+func TestMixed(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	dirs := MakeTestDirs(t, 2)
+	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=:memory:dir3"
+	name := "TestCombineMixed"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":dir1",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "combine"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+		},
+	})
+}
+
+// MakeTestDirs makes directories in /tmp for testing
+func MakeTestDirs(t *testing.T, n int) (dirs []string) {
+	for i := 1; i <= n; i++ {
+		dir := t.TempDir()
+		dirs = append(dirs, dir)
+	}
+	return dirs
+}

backend/compress/compress.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -29,6 +28,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 )
@@ -53,7 +53,7 @@ const (
 	Gzip         = 2
 )
 
-var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
+var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`)
 
 // Register with Fs
 func init() {
@@ -70,6 +70,9 @@ func init() {
 		Name:        "compress",
 		Description: "Compress a remote",
 		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to compress.",
@@ -87,7 +90,7 @@ Generally -1 (default, equivalent to 5) is recommended.
 Levels 1 to 9 increase compression at the cost of speed. Going past 6 
 generally offers very little return.
 
-Level -2 uses Huffmann encoding only. Only use if you know what you
+Level -2 uses Huffman encoding only. Only use if you know what you
 are doing.
 Level 0 turns off compression.`,
 			Default:  sgzip.DefaultCompression,
@@ -127,7 +130,7 @@ type Fs struct {
 	features *fs.Features // optional features
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -180,6 +183,9 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		SetTier:                 true,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -222,7 +228,7 @@ func processFileName(compressedFileName string) (origFileName string, extension
 	// Separate the filename and size from the extension
 	extensionPos := strings.LastIndex(compressedFileName, ".")
 	if extensionPos == -1 {
-		return "", "", 0, errors.New("File name has no extension")
+		return "", "", 0, errors.New("file name has no extension")
 	}
 	extension = compressedFileName[extensionPos:]
 	nameWithSize := compressedFileName[:extensionPos]
@@ -231,11 +237,11 @@ func processFileName(compressedFileName string) (origFileName string, extension
 	}
 	match := nameRegexp.FindStringSubmatch(nameWithSize)
 	if match == nil || len(match) != 3 {
-		return "", "", 0, errors.New("Invalid filename")
+		return "", "", 0, errors.New("invalid filename")
 	}
 	size, err := base64ToInt64(match[2])
 	if err != nil {
-		return "", "", 0, errors.New("Could not decode size")
+		return "", "", 0, errors.New("could not decode size")
 	}
 	return match[1], gzFileExt, size, nil
 }
@@ -304,7 +310,7 @@ func (f *Fs) processEntries(entries fs.DirEntries) (newEntries fs.DirEntries, er
 		case fs.Directory:
 			f.addDir(&newEntries, x)
 		default:
-			return nil, fmt.Errorf("Unknown object type %T", entry)
+			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
 	}
 	return newEntries, nil
@@ -361,13 +367,16 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	meta := readMetadata(ctx, mo)
-	if meta == nil {
-		return nil, errors.New("error decoding metadata")
+	meta, err := readMetadata(ctx, mo)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding metadata: %w", err)
 	}
 	// Create our Object
 	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
-	return f.newObject(o, mo, meta), err
+	if err != nil {
+		return nil, err
+	}
+	return f.newObject(o, mo, meta), nil
 }
 
 // checkCompressAndType checks if an object is compressible and determines it's mime type
@@ -445,7 +454,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 		return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
 	}
 
-	// Need to include what we allready read
+	// Need to include what we already read
 	in = &ReadCloserWrapper{
 		Reader: io.MultiReader(bytes.NewReader(buf), in),
 		Closer: in,
@@ -458,7 +467,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := ioutil.TempFile("", "rclone-press-")
+	tempFile, err := os.CreateTemp("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -466,10 +475,10 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 		_ = os.Remove(tempFile.Name())
 	}()
 	if err != nil {
-		return nil, fmt.Errorf("Failed to create temporary local FS to spool file: %w", err)
+		return nil, fmt.Errorf("failed to create temporary local FS to spool file: %w", err)
 	}
 	if _, err = io.Copy(tempFile, in); err != nil {
-		return nil, fmt.Errorf("Failed to write temporary local file: %w", err)
+		return nil, fmt.Errorf("failed to write temporary local file: %w", err)
 	}
 	if _, err = tempFile.Seek(0, 0); err != nil {
 		return nil, err
@@ -536,8 +545,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)
@@ -671,7 +680,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		}
 		return nil, err
 	}
-	return f.newObject(dataObject, mo, meta), err
+	return f.newObject(dataObject, mo, meta), nil
 }
 
 // Put in to the remote path with the modTime given of the given size
@@ -720,23 +729,23 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	if found && (oldObj.(*Object).meta.Mode != Uncompressed || compressible) {
 		err = oldObj.(*Object).Object.Remove(ctx)
 		if err != nil {
-			return nil, fmt.Errorf("Could remove original object: %w", err)
+			return nil, fmt.Errorf("couldn't remove original object: %w", err)
 		}
 	}
 
 	// If our new object is compressed we have to rename it with the correct size.
-	// Uncompressed objects don't store the size in the name so we they'll allready have the correct name.
+	// Uncompressed objects don't store the size in the name so we they'll already have the correct name.
 	if compressible {
 		wrapObj, err := operations.Move(ctx, f.Fs, nil, f.dataName(src.Remote(), newObj.size, compressible), newObj.Object)
 		if err != nil {
-			return nil, fmt.Errorf("Couldn't rename streamed Object.: %w", err)
+			return nil, fmt.Errorf("couldn't rename streamed object: %w", err)
 		}
 		newObj.Object = wrapObj
 	}
 	return newObj, nil
 }
 
-// Temporarely disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
+// Temporarily disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
 // will break stuff. Right no I can't think of a way to make this work.
 
 // PutUnchecked uploads the object
@@ -779,9 +788,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -829,9 +838,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -906,7 +915,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.Fs.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp: not supported by underlying remote")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -915,7 +924,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("can't About: not supported by underlying remote")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1034,24 +1043,19 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim
 }
 
 // This function will read the metadata from a metadata object.
-func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) {
+func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
 	// Open our meradata object
 	rc, err := mo.Open(ctx)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-	defer func() {
-		err := rc.Close()
-		if err != nil {
-			fs.Errorf(mo, "Error closing object: %v", err)
-		}
-	}()
+	defer fs.CheckClose(rc, &err)
 	jr := json.NewDecoder(rc)
 	meta = new(ObjectMetadata)
 	if err = jr.Decode(meta); err != nil {
-		return nil
+		return nil, err
 	}
-	return meta
+	return meta, nil
 }
 
 // Remove removes this object
@@ -1096,6 +1100,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	origName := o.Remote()
 	if o.meta.Mode != Uncompressed || compressible {
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType)
+		if err != nil {
+			return err
+		}
 		if newObject.Object.Remote() != o.Object.Remote() {
 			if removeErr := o.Object.Remove(ctx); removeErr != nil {
 				return removeErr
@@ -1109,9 +1116,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		// If we are, just update the object and metadata
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType)
-	}
-	if err != nil {
-		return err
+		if err != nil {
+			return err
+		}
 	}
 	// Update object metadata and return
 	o.Object = newObject.Object
@@ -1122,6 +1129,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified.
 func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object {
+	if o == nil {
+		log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1134,6 +1144,9 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object
 
 // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand.
 func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object {
+	if o == nil {
+		log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1161,7 +1174,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) {
 		return err
 	}
 	if o.meta == nil {
-		o.meta = readMetadata(ctx, o.mo)
+		o.meta, err = readMetadata(ctx, o.mo)
 	}
 	return err
 }
@@ -1214,6 +1227,21 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return o.meta.MimeType
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	err := o.loadMetadataIfNotLoaded(ctx)
+	if err != nil {
+		return nil, err
+	}
+	do, ok := o.mo.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
@@ -1360,6 +1388,51 @@ func (o *ObjectInfo) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	return "", nil // cannot know the checksum
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *ObjectInfo) ID() string {
+	do, ok := o.src.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+func (o *ObjectInfo) MimeType(ctx context.Context) string {
+	do, ok := o.src.(fs.MimeTyper)
+	if !ok {
+		return ""
+	}
+	return do.MimeType(ctx)
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *ObjectInfo) UnWrap() fs.Object {
+	return fs.UnWrapObjectInfo(o.src)
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.src.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *ObjectInfo) GetTier() string {
+	do, ok := o.src.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
 // ID returns the ID of the Object if known, or "" if not
 func (o *Object) ID() string {
 	do, ok := o.Object.(fs.IDer)
@@ -1412,11 +1485,6 @@ var (
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.FullObjectInfo  = (*ObjectInfo)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/compress/compress_test.go

@@ -61,5 +61,6 @@ func TestRemoteGzip(t *testing.T) {
 			{Name: name, Key: "remote", Value: tempdir},
 			{Name: name, Key: "compression_mode", Value: "gzip"},
 		},
+		QuickTestOK: true,
 	})
 }

backend/crypt/cipher.go

@@ -96,7 +96,7 @@ func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
 	case "obfuscate":
 		mode = NameEncryptionObfuscated
 	default:
-		err = fmt.Errorf("Unknown file name encryption mode %q", s)
+		err = fmt.Errorf("unknown file name encryption mode %q", s)
 	}
 	return mode, err
 }
@@ -127,11 +127,11 @@ type fileNameEncoding interface {
 // RFC4648
 //
 // The standard encoding is modified in two ways
-//  * it becomes lower case (no-one likes upper case filenames!)
-//  * we strip the padding character `=`
+//   - it becomes lower case (no-one likes upper case filenames!)
+//   - we strip the padding character `=`
 type caseInsensitiveBase32Encoding struct{}
 
-// EncodeToString encodes a strign using the modified version of
+// EncodeToString encodes a string using the modified version of
 // base32 encoding.
 func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
 	encoded := base32.HexEncoding.EncodeToString(src)
@@ -162,7 +162,7 @@ func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
 	case "base32768":
 		enc = base32768.SafeEncoding
 	default:
-		err = fmt.Errorf("Unknown file name encoding mode %q", s)
+		err = fmt.Errorf("unknown file name encoding mode %q", s)
 	}
 	return enc, err
 }
@@ -244,7 +244,7 @@ func (c *Cipher) putBlock(buf []byte) {
 
 // encryptSegment encrypts a path segment
 //
-// This uses EME with AES
+// This uses EME with AES.
 //
 // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
@@ -254,8 +254,8 @@ func (c *Cipher) putBlock(buf []byte) {
 // same filename must encrypt to the same thing.
 //
 // This means that
-//  * filenames with the same name will encrypt the same
-//  * filenames which start the same won't have a common prefix
+//   - filenames with the same name will encrypt the same
+//   - filenames which start the same won't have a common prefix
 func (c *Cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
@@ -1085,7 +1085,7 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 
 // DecryptDataSeek decrypts the data stream from offset
 //
-// The open function must return a ReadCloser opened to the offset supplied
+// The open function must return a ReadCloser opened to the offset supplied.
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
 func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {

backend/crypt/cipher_test.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -1073,7 +1072,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1144,15 +1143,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := ioutil.ReadAll(encrypted)
+		out, err := io.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
+		decrypted, err := c.DecryptData(io.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = ioutil.ReadAll(decrypted)
+		out, err = io.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1187,7 +1186,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1257,12 +1256,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := ioutil.NopCloser(io.MultiReader(in1, in2))
+	in := io.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1274,14 +1273,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
+	plaintext, err := io.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := ioutil.ReadAll(encrypted)
+	ciphertext, err := io.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1300,7 +1299,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1490,7 +1489,7 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = ioutil.ReadAll(fh)
+		_, err = io.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
@@ -1514,7 +1513,7 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = ioutil.ReadAll(fh)
+	_, err = io.ReadAll(fh)
 	assert.Error(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
@@ -1524,13 +1523,13 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
 			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = ioutil.ReadAll(fh)
+			_, err = io.ReadAll(fh)
 			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
 		}
 		file16copy[i] ^= 0xFF
@@ -1565,7 +1564,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := ioutil.ReadAll(fh)
+	out, err := io.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)

backend/crypt/crypt.go

@@ -28,6 +28,9 @@ func init() {
 		Description: "Encrypt/Decrypt a remote",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to encrypt/decrypt.\n\nNormally should contain a ':' and a path, e.g. \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -122,7 +125,7 @@ names, or for debugging purposes.`,
 
 This option could help with shortening the encrypted filename. The 
 suitable option would depend on the way your remote count the filename
-length and if it's case sensitve.`,
+length and if it's case sensitive.`,
 			Default: "base32",
 			Examples: []fs.OptionExample{
 				{
@@ -241,6 +244,9 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		SetTier:                 true,
 		GetTier:                 true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -328,7 +334,7 @@ func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntr
 		case fs.Directory:
 			f.addDir(ctx, &newEntries, x)
 		default:
-			return nil, fmt.Errorf("Unknown object type %T", entry)
+			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
 	}
 	return newEntries, nil
@@ -501,9 +507,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -526,9 +532,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -597,7 +603,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 func (f *Fs) CleanUp(ctx context.Context) error {
 	do := f.Fs.Features().CleanUp
 	if do == nil {
-		return errors.New("can't CleanUp")
+		return errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -606,7 +612,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
-		return nil, errors.New("About not supported")
+		return nil, errors.New("not supported by underlying remote")
 	}
 	return do(ctx)
 }
@@ -1056,6 +1062,50 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	return "", nil
 }
 
+// GetTier returns storage tier or class of the Object
+func (o *ObjectInfo) GetTier() string {
+	do, ok := o.ObjectInfo.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *ObjectInfo) ID() string {
+	do, ok := o.ObjectInfo.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.ObjectInfo.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+//
+// This is deliberately unsupported so we don't leak mime type info by
+// default.
+func (o *ObjectInfo) MimeType(ctx context.Context) string {
+	return ""
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *ObjectInfo) UnWrap() fs.Object {
+	return fs.UnWrapObjectInfo(o.ObjectInfo)
+}
+
 // ID returns the ID of the Object if known, or "" if not
 func (o *Object) ID() string {
 	do, ok := o.Object.(fs.IDer)
@@ -1084,6 +1134,26 @@ func (o *Object) GetTier() string {
 	return do.GetTier()
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.Object.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+//
+// This is deliberately unsupported so we don't leak mime type info by
+// default.
+func (o *Object) MimeType(ctx context.Context) string {
+	return ""
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1106,10 +1176,6 @@ var (
 	_ fs.UserInfoer      = (*Fs)(nil)
 	_ fs.Disconnecter    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
+	_ fs.FullObjectInfo  = (*ObjectInfo)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/crypt/crypt_internal_test.go

@@ -91,7 +91,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	src := f.newObjectInfo(oi, nonce)
 
 	// Test ObjectInfo methods
-	assert.Equal(t, int64(outBuf.Len()), src.Size())
+	if !f.opt.NoDataEncryption {
+		assert.Equal(t, int64(outBuf.Len()), src.Size())
+	}
 	assert.Equal(t, f, src.Fs())
 	assert.NotEqual(t, path, src.Remote())
 

backend/crypt/crypt_test.go

@@ -4,6 +4,7 @@ package crypt_test
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/rclone/rclone/backend/crypt"
@@ -46,6 +47,7 @@ func TestStandardBase32(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -67,6 +69,7 @@ func TestStandardBase64(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -88,6 +91,7 @@ func TestStandardBase32768(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -109,6 +113,7 @@ func TestOff(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -117,6 +122,9 @@ func TestObfuscate(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
+	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt3"
 	fstests.Run(t, &fstests.Opt{
@@ -131,6 +139,7 @@ func TestObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }
 
@@ -139,6 +148,9 @@ func TestNoDataObfuscate(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
+	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt4"
 	fstests.Run(t, &fstests.Opt{
@@ -154,5 +166,6 @@ func TestNoDataObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
+		QuickTestOK:                  true,
 	})
 }

backend/crypt/pkcs7/pkcs7.go

@@ -8,11 +8,11 @@ import "errors"
 
 // Errors Unpad can return
 var (
-	ErrorPaddingNotFound      = errors.New("Bad PKCS#7 padding - not padded")
-	ErrorPaddingNotAMultiple  = errors.New("Bad PKCS#7 padding - not a multiple of blocksize")
-	ErrorPaddingTooLong       = errors.New("Bad PKCS#7 padding - too long")
-	ErrorPaddingTooShort      = errors.New("Bad PKCS#7 padding - too short")
-	ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same")
+	ErrorPaddingNotFound      = errors.New("bad PKCS#7 padding - not padded")
+	ErrorPaddingNotAMultiple  = errors.New("bad PKCS#7 padding - not a multiple of blocksize")
+	ErrorPaddingTooLong       = errors.New("bad PKCS#7 padding - too long")
+	ErrorPaddingTooShort      = errors.New("bad PKCS#7 padding - too short")
+	ErrorPaddingNotAllTheSame = errors.New("bad PKCS#7 padding - not all the same")
 )
 
 // Pad buf using PKCS#7 to a multiple of n.

backend/drive/drive.go

@@ -14,10 +14,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"path"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -50,6 +51,7 @@ import (
 	drive_v2 "google.golang.org/api/drive/v2"
 	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
 )
 
 // Constants
@@ -70,7 +72,7 @@ const (
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	minChunkSize     = fs.SizeSuffix(googleapi.MinUploadChunkSize)
 	defaultChunkSize = 8 * fs.Mebi
-	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
+	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
 	defaultXDGIcon   = "text-html"
@@ -276,6 +278,7 @@ Leave blank normally.
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 `,
+			Advanced: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
@@ -564,6 +567,27 @@ If this is set then rclone will not show any dangling shortcuts in listings.
 `,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name: "resource_key",
+			Help: `Resource key for accessing a link-shared file.
+
+If you need to access files shared with a link like this
+
+    https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing
+
+Then you will need to use the first part "XXX" as the "root_folder_id"
+and the second part "YYY" as the "resource_key" otherwise you will get
+404 not found errors when trying to access the directory.
+
+See: https://developers.google.com/drive/api/guides/resource-keys
+
+This resource key requirement only applies to a subset of old files.
+
+Note also that opening the folder once in the web interface (with the
+user you've authenticated rclone with) seems to be enough so that the
+resource key is no needed.
+`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -625,6 +649,7 @@ type Options struct {
 	StopOnDownloadLimit       bool                 `config:"stop_on_download_limit"`
 	SkipShortcuts             bool                 `config:"skip_shortcuts"`
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
+	ResourceKey               string               `config:"resource_key"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -650,6 +675,7 @@ type Fs struct {
 	grouping         int32               // number of IDs to search at once in ListR - read with atomic
 	listRmu          *sync.Mutex         // protects listRempties
 	listRempties     map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
+	dirResourceKeys  *sync.Map           // map directory ID to resource key
 }
 
 type baseObject struct {
@@ -660,6 +686,7 @@ type baseObject struct {
 	mimeType     string   // The object MIME type
 	bytes        int64    // size of the object
 	parents      []string // IDs of the parent directories
+	resourceKey  *string  // resourceKey is needed for link shared objects
 }
 type documentObject struct {
 	baseObject
@@ -731,6 +758,9 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
+			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
+				fs.Errorf(f, "Received upload limit error: %v", err)
+				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
 				fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
 				return false, fserrors.FatalError(err)
@@ -800,6 +830,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
 	// If we need to list file inside those shared folders, we must search it without sharedWithMe
 	parentsQuery := bytes.NewBufferString("(")
+	var resourceKeys []string
 	for _, dirID := range dirIDs {
 		if dirID == "" {
 			continue
@@ -820,7 +851,12 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		} else {
 			_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
 		}
+		resourceKey, hasResourceKey := f.dirResourceKeys.Load(dirID)
+		if hasResourceKey {
+			resourceKeys = append(resourceKeys, fmt.Sprintf("%s/%s", dirID, resourceKey))
+		}
 	}
+	resourceKeysHeader := strings.Join(resourceKeys, ",")
 	if parentsQuery.Len() > 1 {
 		_ = parentsQuery.WriteByte(')')
 		query = append(query, parentsQuery.String())
@@ -829,8 +865,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	if title != "" {
 		searchTitle := f.opt.Enc.FromStandardName(title)
 		// Escaping the backslash isn't documented but seems to work
-		searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
-		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
+		searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
+		searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
 
 		var titleQuery bytes.Buffer
 		_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
@@ -884,7 +920,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	}
 	list.SupportsAllDrives(true)
 	list.IncludeItemsFromAllDrives(true)
-	if f.isTeamDrive {
+	if f.isTeamDrive && !f.opt.SharedWithMe {
 		list.DriveId(f.opt.TeamDriveID)
 		list.Corpora("drive")
 	}
@@ -892,6 +928,10 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	if f.rootFolderID == "appDataFolder" {
 		list.Spaces("appDataFolder")
 	}
+	// Add resource Keys if necessary
+	if resourceKeysHeader != "" {
+		list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader)
+	}
 
 	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields)
 
@@ -1068,7 +1108,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -1151,15 +1191,16 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:         name,
-		root:         root,
-		opt:          *opt,
-		ci:           ci,
-		pacer:        fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
-		m:            m,
-		grouping:     listRGrouping,
-		listRmu:      new(sync.Mutex),
-		listRempties: make(map[string]struct{}),
+		name:            name,
+		root:            root,
+		opt:             *opt,
+		ci:              ci,
+		pacer:           fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
+		m:               m,
+		grouping:        listRGrouping,
+		listRmu:         new(sync.Mutex),
+		listRempties:    make(map[string]struct{}),
+		dirResourceKeys: new(sync.Map),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.fileFields = f.getFileFields()
@@ -1169,17 +1210,18 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
-	f.svc, err = drive.New(f.client)
+	f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create Drive client: %w", err)
 	}
 
 	if f.opt.V2DownloadMinSize >= 0 {
-		f.v2Svc, err = drive_v2.New(f.client)
+		f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
 		if err != nil {
 			return nil, fmt.Errorf("couldn't create Drive v2 client: %w", err)
 		}
@@ -1221,6 +1263,11 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 
 	f.dirCache = dircache.New(f.root, f.rootFolderID, f)
 
+	// If resource key is set then cache it for the root folder id
+	if f.opt.ResourceKey != "" {
+		f.dirResourceKeys.Store(f.rootFolderID, f.opt.ResourceKey)
+	}
+
 	// Parse extensions
 	if f.opt.Extensions != "" {
 		if f.opt.ExportExtensions != defaultExportExtensions {
@@ -1319,12 +1366,16 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
 			}
 		}
 	}
-	return &Object{
+	o := &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
 		md5sum:     strings.ToLower(info.Md5Checksum),
 		v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
 	}
+	if info.ResourceKey != "" {
+		o.resourceKey = &info.ResourceKey
+	}
+	return o
 }
 
 // newDocumentObject creates an fs.Object for a google docs drive.File
@@ -2014,7 +2065,7 @@ func splitID(compositeID string) (actualID, shortcutID string) {
 
 // isShortcutID returns true if compositeID refers to a shortcut
 func isShortcutID(compositeID string) bool {
-	return strings.IndexRune(compositeID, shortcutSeparator) >= 0
+	return strings.ContainsRune(compositeID, shortcutSeparator)
 }
 
 // actualID returns an actual ID from a composite ID
@@ -2085,6 +2136,10 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File
 	case item.MimeType == driveFolderType:
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, item.Id)
+		// cache the resource key for later lookups
+		if item.ResourceKey != "" {
+			f.dirResourceKeys.Store(item.Id, item.ResourceKey)
+		}
 		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
 		d := fs.NewDir(remote, when).SetID(item.Id)
 		if len(item.Parents) > 0 {
@@ -2126,7 +2181,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -2168,10 +2223,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 			exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
 			if exportExt == "" {
-				return nil, fmt.Errorf("No export format found for %q", importMimeType)
+				return nil, fmt.Errorf("no export format found for %q", importMimeType)
 			}
 			if exportExt != srcExt && !f.opt.AllowImportNameChange {
-				return nil, fmt.Errorf("Can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt)
+				return nil, fmt.Errorf("can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt)
 			}
 		}
 	}
@@ -2360,9 +2415,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2429,11 +2484,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Copy(id, createInfo).
+		copy := f.svc.Files.Copy(id, createInfo).
 			Fields(partialFields).
 			SupportsAllDrives(true).
-			KeepRevisionForever(f.opt.KeepRevisionForever).
-			Context(ctx).Do()
+			KeepRevisionForever(f.opt.KeepRevisionForever)
+		srcObj.addResourceKey(copy.Header())
+		info, err = copy.Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -2475,7 +2531,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // result of List()
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	if f.opt.TrashedOnly {
-		return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files")
+		return errors.New("can't purge with --drive-trashed-only, use delete if you want to selectively delete files")
 	}
 	return f.purgeCheck(ctx, dir, false)
 }
@@ -2594,9 +2650,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2941,12 +2997,12 @@ func (f *Fs) changeServiceAccountFile(ctx context.Context, file string) (err err
 		return fmt.Errorf("drive: failed when making oauth client: %w", err)
 	}
 	f.client = oAuthClient
-	f.svc, err = drive.New(f.client)
+	f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
 	if err != nil {
 		return fmt.Errorf("couldn't create Drive client: %w", err)
 	}
 	if f.opt.V2DownloadMinSize >= 0 {
-		f.v2Svc, err = drive_v2.New(f.client)
+		f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
 		if err != nil {
 			return fmt.Errorf("couldn't create Drive v2 client: %w", err)
 		}
@@ -3235,7 +3291,7 @@ This will return a JSON list of objects like this
 
 With the -o config parameter it will output the list in a format
 suitable for adding to a config file to make aliases for all the
-drives found.
+drives found and a combined drive.
 
     [My Drive]
     type = alias
@@ -3245,10 +3301,15 @@ drives found.
     type = alias
     remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
 
-Adding this to the rclone config file will cause those team drives to
-be accessible with the aliases shown. This may require manual editing
-of the names.
+    [AllDrives]
+    type = combine
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
+Adding this to the rclone config file will cause those team drives to
+be accessible with the aliases shown. Any illegal characters will be
+substituted with "_" and duplicate names will have numbers suffixed.
+It will also add a remote called AllDrives which shows all the shared
+drives combined into one directory tree.
 `,
 }, {
 	Name:  "untrash",
@@ -3296,6 +3357,12 @@ attempted if possible.
 
 Use the -i flag to see what would be copied before copying.
 `,
+}, {
+	Name:  "exportformats",
+	Short: "Dump the export formats for debug purposes",
+}, {
+	Name:  "importformats",
+	Short: "Dump the import formats for debug purposes",
 }}
 
 // Command the backend to run a named command
@@ -3315,7 +3382,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			out["service_account_file"] = f.opt.ServiceAccountFile
 		}
 		if _, ok := opt["chunk_size"]; ok {
-			out["chunk_size"] = fmt.Sprintf("%s", f.opt.ChunkSize)
+			out["chunk_size"] = f.opt.ChunkSize.String()
 		}
 		return out, nil
 	case "set":
@@ -3332,11 +3399,11 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		}
 		if chunkSize, ok := opt["chunk_size"]; ok {
 			chunkSizeMap := make(map[string]string)
-			chunkSizeMap["previous"] = fmt.Sprintf("%s", f.opt.ChunkSize)
+			chunkSizeMap["previous"] = f.opt.ChunkSize.String()
 			if err = f.changeChunkSize(chunkSize); err != nil {
 				return out, err
 			}
-			chunkSizeString := fmt.Sprintf("%s", f.opt.ChunkSize)
+			chunkSizeString := f.opt.ChunkSize.String()
 			f.m.Set("chunk_size", chunkSizeString)
 			chunkSizeMap["current"] = chunkSizeString
 			out["chunk_size"] = chunkSizeMap
@@ -3364,14 +3431,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
+		re := regexp.MustCompile(`[^\w_. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
-			for _, drive := range drives {
+			upstreams := []string{}
+			names := make(map[string]struct{}, len(drives))
+			for i, drive := range drives {
+				name := re.ReplaceAllString(drive.Name, "_")
+				for {
+					if _, found := names[name]; !found {
+						break
+					}
+					name += fmt.Sprintf("-%d", i)
+				}
+				names[name] = struct{}{}
 				lines = append(lines, "")
-				lines = append(lines, fmt.Sprintf("[%s]", drive.Name))
-				lines = append(lines, fmt.Sprintf("type = alias"))
+				lines = append(lines, fmt.Sprintf("[%s]", name))
+				lines = append(lines, "type = alias")
 				lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id))
+				upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name))
 			}
+			lines = append(lines, "")
+			lines = append(lines, "[AllDrives]")
+			lines = append(lines, "type = combine")
+			lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " ")))
 			return lines, nil
 		}
 		return drives, nil
@@ -3394,6 +3477,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			}
 		}
 		return nil, nil
+	case "exportformats":
+		return f.exportFormats(ctx), nil
+	case "importformats":
+		return f.importFormats(ctx), nil
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -3443,12 +3530,6 @@ func (o *baseObject) Size() int64 {
 	return o.bytes
 }
 
-// getRemoteInfo returns a drive.File for the remote
-func (f *Fs) getRemoteInfo(ctx context.Context, remote string) (info *drive.File, err error) {
-	info, _, _, _, _, err = f.getRemoteInfoWithExport(ctx, remote)
-	return
-}
-
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
@@ -3489,7 +3570,6 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *baseObject) ModTime(ctx context.Context) time.Time {
@@ -3530,6 +3610,14 @@ func (o *baseObject) Storable() bool {
 	return true
 }
 
+// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
+// object if required.
+func (o *baseObject) addResourceKey(header http.Header) {
+	if o.resourceKey != nil {
+		header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
+	}
+}
+
 // httpResponse gets an http.Response object for the object
 // using the url and method passed in
 func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
@@ -3545,6 +3633,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 		// Don't supply range requests for 0 length objects as they always fail
 		delete(req.Header, "Range")
 	}
+	o.addResourceKey(req.Header)
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {
@@ -3624,7 +3713,7 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
 				url += "acknowledgeAbuse=true"
 				_, res, err = o.httpResponse(ctx, url, "GET", options)
 			} else {
-				err = fmt.Errorf("Use the --drive-acknowledge-abuse flag to download this file: %w", err)
+				err = fmt.Errorf("use the --drive-acknowledge-abuse flag to download this file: %w", err)
 			}
 		}
 		if err != nil {
@@ -3711,7 +3800,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(data)), nil
+	return io.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
@@ -3737,7 +3826,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/drive/drive_internal_test.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -19,6 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/sync"
@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
 )
 
 func TestDriveScopes(t *testing.T) {
@@ -76,7 +77,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -190,6 +191,60 @@ func TestExtensionsForImportFormats(t *testing.T) {
 	}
 }
 
+func (f *Fs) InternalTestShouldRetry(t *testing.T) {
+	ctx := context.Background()
+	gatewayTimeout := googleapi.Error{
+		Code: 503,
+	}
+	timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout)
+	assert.True(t, timeoutRetry)
+	assert.Equal(t, &gatewayTimeout, timeoutError)
+	generic403 := googleapi.Error{
+		Code: 403,
+	}
+	rLEItem := googleapi.ErrorItem{
+		Reason:  "rateLimitExceeded",
+		Message: "User rate limit exceeded.",
+	}
+	generic403.Errors = append(generic403.Errors, rLEItem)
+	oldStopUpload := f.opt.StopOnUploadLimit
+	oldStopDownload := f.opt.StopOnDownloadLimit
+	f.opt.StopOnUploadLimit = true
+	f.opt.StopOnDownloadLimit = true
+	defer func() {
+		f.opt.StopOnUploadLimit = oldStopUpload
+		f.opt.StopOnDownloadLimit = oldStopDownload
+	}()
+	expectedRLError := fserrors.FatalError(&generic403)
+	rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403)
+	assert.False(t, rateLimitRetry)
+	assert.Equal(t, rateLimitErr, expectedRLError)
+	dQEItem := googleapi.ErrorItem{
+		Reason: "downloadQuotaExceeded",
+	}
+	generic403.Errors[0] = dQEItem
+	expectedDQError := fserrors.FatalError(&generic403)
+	downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, downloadQuotaRetry)
+	assert.Equal(t, downloadQuotaError, expectedDQError)
+	tDFLEItem := googleapi.ErrorItem{
+		Reason: "teamDriveFileLimitExceeded",
+	}
+	generic403.Errors[0] = tDFLEItem
+	expectedTDFLError := fserrors.FatalError(&generic403)
+	teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, teamDriveFileLimitRetry)
+	assert.Equal(t, teamDriveFileLimitError, expectedTDFLError)
+	qEItem := googleapi.ErrorItem{
+		Reason: "quotaExceeded",
+	}
+	generic403.Errors[0] = qEItem
+	expectedQuotaError := fserrors.FatalError(&generic403)
+	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, quotaExceededRetry)
+	assert.Equal(t, quotaExceededError, expectedQuotaError)
+}
+
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	oldAllow := f.opt.AllowImportNameChange
 	f.opt.AllowImportNameChange = true
@@ -378,9 +433,9 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) {
 	// Make some objects, one in a subdir
 	contents := random.String(100)
 	file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
-	_, obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
+	obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
 	file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
-	_, _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
+	_ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
 
 	// Check objects
 	checkObjects := func() {
@@ -462,6 +517,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
@@ -496,7 +554,7 @@ func (f *Fs) InternalTestAgeQuery(t *testing.T) {
 	require.NoError(t, err)
 
 	file1 := fstest.Item{ModTime: time.Now(), Path: "agequery.txt"}
-	_, _ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
+	_ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
 
 	// validate sync/copy
 	const timeQuery = "(modifiedTime >= '"
@@ -545,6 +603,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
+	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/dropbox/batcher.go

@@ -118,12 +118,12 @@ func (b *batcher) Batching() bool {
 }
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
+func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
 	var arg = &files.UploadSessionFinishBatchArg{
 		Entries: items,
 	}
 	err = b.f.pacer.Call(func() (bool, error) {
-		batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
+		complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
 		// If error is insufficient space then don't retry
 		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -137,7 +137,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
 	if err != nil {
 		return nil, fmt.Errorf("batch commit failed: %w", err)
 	}
-	return batchStatus, nil
+	return complete, nil
 }
 
 // finishBatchJobStatus waits for the batch to complete returning completed entries
@@ -199,26 +199,11 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
 	fs.Debugf(b.f, "Committing %s", desc)
 
 	// finalise the batch getting either a result or a job id to poll
-	batchStatus, err := b.finishBatch(ctx, items)
+	complete, err := b.finishBatch(ctx, items)
 	if err != nil {
 		return err
 	}
 
-	// check whether batch is complete
-	var complete *files.UploadSessionFinishBatchResult
-	switch batchStatus.Tag {
-	case "async_job_id":
-		// wait for batch to complete
-		complete, err = b.finishBatchJobStatus(ctx, batchStatus)
-		if err != nil {
-			return err
-		}
-	case "complete":
-		complete = batchStatus.Complete
-	default:
-		return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
-	}
-
 	// Check we got the right number of entries
 	entries := complete.Entries
 	if len(entries) != len(results) {
@@ -319,9 +304,12 @@ outer:
 //
 // Can be called from atexit handler
 func (b *batcher) Shutdown() {
+	if !b.Batching() {
+		return
+	}
 	b.shutOnce.Do(func() {
 		atexit.Unregister(b.atexit)
-		fs.Infof(b.f, "Commiting uploads - please wait...")
+		fs.Infof(b.f, "Committing uploads - please wait...")
 		// show that batcher is shutting down
 		close(b.closed)
 		// quit the commitLoop by sending a quitRequest message

backend/dropbox/dropbox.go

@@ -268,7 +268,7 @@ default based on the batch_mode in use.
 			Advanced: true,
 		}, {
 			Name:     "batch_commit_timeout",
-			Help:     `Max time to wait for a batch to finish comitting`,
+			Help:     `Max time to wait for a batch to finish committing`,
 			Default:  fs.Duration(10 * time.Minute),
 			Advanced: true,
 		}, {
@@ -472,10 +472,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		args := team.NewMembersGetInfoArgs(members)
 
 		memberIds, err := f.team.MembersGetInfo(args)
-
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
+		if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil {
+			return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate)
+		}
 
 		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
@@ -923,7 +925,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1042,9 +1044,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1103,9 +1105,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1197,7 +1199,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 			return
 		}
 		if len(listRes.Links) == 0 {
-			err = errors.New("Dropbox says the sharing link already exists, but list came back empty")
+			err = errors.New("sharing link already exists, but list came back empty")
 			return
 		}
 		linkRes = listRes.Links[0]
@@ -1209,7 +1211,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		case *sharing.FolderLinkMetadata:
 			link = res.Url
 		default:
-			err = fmt.Errorf("Don't know how to extract link, response has unknown format: %T", res)
+			err = fmt.Errorf("don't know how to extract link, response has unknown format: %T", res)
 		}
 	}
 	return
@@ -1269,7 +1271,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("about failed: %w", err)
+		return nil, err
 	}
 	var total uint64
 	if q.Allocation != nil {
@@ -1370,10 +1372,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 
 	if timeout < 30 {
 		timeout = 30
+		fs.Debugf(f, "Increasing poll interval to minimum 30s")
 	}
 
 	if timeout > 480 {
 		timeout = 480
+		fs.Debugf(f, "Decreasing poll interval to maximum 480s")
 	}
 
 	err = f.pacer.Call(func() (bool, error) {
@@ -1431,7 +1435,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			}
 
 			if entryPath != "" {
-				notifyFunc(entryPath, entryType)
+				notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
 			}
 		}
 		if !changeList.HasMore {
@@ -1665,7 +1669,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 						correctOffset := uErr.EndpointError.IncorrectOffset.CorrectOffset
 						delta := int64(correctOffset) - int64(cursor.Offset)
 						skip += delta
-						what := fmt.Sprintf("incorrect offset error receved: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
+						what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
 						if skip < 0 {
 							return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
 						} else if skip == chunkSize {
@@ -1693,6 +1697,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 		if size > 0 {
 			// if size is known, check if next chunk is final
 			appendArg.Close = uint64(size)-in.BytesRead() <= uint64(chunkSize)
+			if in.BytesRead() > uint64(size) {
+				return nil, fmt.Errorf("expected %d bytes in input, but have read %d so far", size, in.BytesRead())
+			}
 		} else {
 			// if size is unknown, upload as long as we can read full chunks from the reader
 			appendArg.Close = in.BytesRead()-cursor.Offset < uint64(chunkSize)
@@ -1756,7 +1763,7 @@ func checkPathLength(name string) (err error) {
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/fichier/api.go

@@ -28,25 +28,44 @@ var retryErrorCodes = []int{
 	509, // Bandwidth Limit Exceeded
 }
 
+var errorRegex = regexp.MustCompile(`#\d{1,3}`)
+
+func parseFichierError(err error) int {
+	matches := errorRegex.FindStringSubmatch(err.Error())
+	if len(matches) == 0 {
+		return 0
+	}
+	code, err := strconv.Atoi(matches[0])
+	if err != nil {
+		fs.Debugf(nil, "failed parsing fichier error: %v", err)
+		return 0
+	}
+	return code
+}
+
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	// Detect this error which the integration tests provoke
-	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
+	// 1Fichier uses HTTP error code 403 (Forbidden) for all kinds of errors with
+	// responses looking like this: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
 	//
-	// https://1fichier.com/api.html
-	//
-	// file/ls.cgi is limited :
-	//
-	// Warning (can be changed in case of abuses) :
-	// List all files of the account is limited to 1 request per hour.
-	// List folders is limited to 5 000 results and 1 request per folder per 30s.
-	if err != nil && strings.Contains(err.Error(), "Flood detected") {
-		fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
-		time.Sleep(30 * time.Second)
+	// We attempt to parse the actual 1Fichier error code from this body and handle it accordingly
+	// Most importantly #374 (Flood detected: IP locked) which the integration tests provoke
+	// The list below is far from complete and should be expanded if we see any more error codes.
+	if err != nil {
+		switch parseFichierError(err) {
+		case 93:
+			return false, err // No such user
+		case 186:
+			return false, err // IP blocked?
+		case 374:
+			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
+			time.Sleep(30 * time.Second)
+		default:
+		}
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -468,7 +487,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 	fileName = f.opt.Enc.FromStandardName(fileName)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
+		return nil, errors.New("invalid UploadID")
 	}
 
 	opts := rest.Opts{
@@ -510,7 +529,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
 	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
+		return nil, errors.New("invalid UploadID")
 	}
 
 	opts := rest.Opts{

backend/fichier/fichier.go

@@ -1,3 +1,4 @@
+// Package fichier provides an interface to the 1Fichier storage system.
 package fichier
 
 import (
@@ -294,7 +295,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 			path, ok := f.dirCache.GetInv(directoryID)
 
 			if !ok {
-				return nil, errors.New("Cannot find dir in dircache")
+				return nil, errors.New("cannot find dir in dircache")
 			}
 
 			return f.newObjectFromFile(ctx, path, file), nil

backend/fichier/structs.go

@@ -84,7 +84,7 @@ type CopyFileResponse struct {
 	URLs    []FileCopy `json:"urls"`
 }
 
-// FileCopy is used in the the CopyFileResponse
+// FileCopy is used in the CopyFileResponse
 type FileCopy struct {
 	FromURL string `json:"from_url"`
 	ToURL   string `json:"to_url"`

backend/filefabric/api/types.go

@@ -19,7 +19,7 @@ const (
 	timeFormatJSON = `"` + timeFormatParameters + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // filefabric API
 type Time time.Time
 
@@ -95,7 +95,7 @@ type Status struct {
 	// Warning string `json:"warning"` // obsolete
 }
 
-// Status statisfies the error interface
+// Status satisfies the error interface
 func (e *Status) Error() string {
 	return fmt.Sprintf("%s (%s)", e.Message, e.Code)
 }

backend/filefabric/filefabric.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -150,7 +149,7 @@ type Fs struct {
 	opt             Options            // parsed options
 	features        *fs.Features       // optional features
 	m               configmap.Mapper   // to save config
-	srv             *rest.Client       // the connection to the one drive server
+	srv             *rest.Client       // the connection to the server
 	dirCache        *dircache.DirCache // Map of directory path to directory id
 	pacer           *fs.Pacer          // pacer for API calls
 	tokenMu         sync.Mutex         // hold when reading the token
@@ -373,7 +372,7 @@ type params map[string]interface{}
 
 // rpc calls the rpc.php method of the SME file fabric
 //
-// This is an entry point to all the method calls
+// This is an entry point to all the method calls.
 //
 // If result is nil then resp.Body will need closing
 func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKError, options []fs.OpenOption) (resp *http.Response, err error) {
@@ -490,7 +489,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 				// Root is a dir - cache its ID
 				f.dirCache.Put(f.root, info.ID)
 			}
-		} else {
+			//} else {
 			// Root is not found so a directory
 		}
 	}
@@ -678,7 +677,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -697,7 +696,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -783,9 +782,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -843,7 +842,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.purgeCheck(ctx, dir, false)
 }
 
-// Wait for the the background task to complete if necessary
+// Wait for the background task to complete if necessary
 func (f *Fs) waitForBackgroundTask(ctx context.Context, taskID api.String) (err error) {
 	if taskID == "" || taskID == "0" {
 		// No task to wait for
@@ -956,9 +955,9 @@ func (f *Fs) move(ctx context.Context, isDir bool, id, oldLeaf, newLeaf, oldDire
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1135,7 +1134,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1187,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{
@@ -1201,7 +1199,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/ftp/ftp.go

@@ -45,7 +45,7 @@ const (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "ftp",
-		Description: "FTP Connection",
+		Description: "FTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "host",
@@ -81,8 +81,22 @@ security from the server in order to upgrade a plain text connection
 to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
-			Name:     "concurrency",
-			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited.",
+			Name: "concurrency",
+			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+
+Note that setting this is very likely to cause deadlocks so it should
+be used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of |--transfers| and |--checkers|.
+
+If you use |--check-first| then it just needs to be one more than the
+maximum of |--checkers| and |--transfers|.
+
+So for |concurrency 3| you'd use |--checkers 2 --transfers 2
+--check-first| or |--checkers 1 --transfers 1|.
+
+`, "|", "`", -1),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -100,11 +114,21 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "disable_utf8",
+			Help:     "Disable using UTF-8 even if server advertises support.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "writing_mdtm",
 			Help:     "Use MDTM to set modification time (VsFtpd quirk)",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "force_list_hidden",
+			Help:     "Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -184,7 +208,9 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
+	DisableUTF8       bool                 `config:"disable_utf8"`
 	WritingMDTM       bool                 `config:"writing_mdtm"`
+	ForceListHidden   bool                 `config:"force_list_hidden"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
@@ -310,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		if err != nil {
+			return nil, err
 		}
-		return
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -325,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -338,12 +388,18 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.DisableMLSD {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
 	}
+	if f.opt.DisableUTF8 {
+		ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
+	}
 	if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
 		ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
 	}
 	if f.opt.WritingMDTM {
 		ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
 	}
+	if f.opt.ForceListHidden {
+		ftpConfig = append(ftpConfig, ftp.DialWithForceListHidden(true))
+	}
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
@@ -478,7 +534,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		protocol = "ftps://"
 	}
 	if opt.TLS && opt.ExplicitTLS {
-		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config")
 	}
 	var tlsConfig *tls.Config
 	if opt.TLS || opt.ExplicitTLS {
@@ -709,7 +765,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	case <-timer.C:
 		// if timer fired assume no error but connection dead
 		fs.Errorf(f, "Timeout when waiting for List")
-		return nil, errors.New("Timeout when waiting for List")
+		return nil, errors.New("timeout when waiting for List")
 	}
 
 	// Annoyingly FTP returns success for a directory which
@@ -760,11 +816,12 @@ func (f *Fs) Hashes() hash.Set {
 
 // Precision shows whether modified time is supported or not depending on the
 // FTP server capabilities, namely whether FTP server:
-// - accepts the MDTM command to get file time (fGetTime)
-//   or supports MLSD returning precise file time in the list (fLstTime)
-// - accepts the MFMT command to set file time (fSetTime)
-//   or non-standard form of the MDTM command (fSetTime, too)
-//   used by VsFtpd for the same purpose (WritingMDTM)
+//   - accepts the MDTM command to get file time (fGetTime)
+//     or supports MLSD returning precise file time in the list (fLstTime)
+//   - accepts the MFMT command to set file time (fSetTime)
+//     or non-standard form of the MDTM command (fSetTime, too)
+//     used by VsFtpd for the same purpose (WritingMDTM)
+//
 // See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
 func (f *Fs) Precision() time.Duration {
 	if (f.fGetTime || f.fLstTime) && f.fSetTime {
@@ -1140,7 +1197,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000             // 100 MiB
-		idleTimeout = 40 * time.Millisecond // small because test server is local
-		maxTime     = 10 * time.Second      // prevent test hangup
+		fileSize    = 100000000        // 100 MiB
+		idleTimeout = 1 * time.Second  // small because test server is local
+		maxTime     = 10 * time.Second // prevent test hangup
 	)
 
 	if testing.Short() {

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,11 +19,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -43,6 +44,7 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
+	option "google.golang.org/api/option"
 
 	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
@@ -304,6 +306,23 @@ rclone does if you know the bucket exists already.
 `,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "decompress",
+			Help: `If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to GCS with "Content-Encoding: gzip"
+set. Normally rclone will download these files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be decompressed.
+`,
+			Advanced: true,
+			Default:  false,
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for the service.\n\nLeave blank normally.",
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -327,21 +346,24 @@ type Options struct {
 	Location                  string               `config:"location"`
 	StorageClass              string               `config:"storage_class"`
 	NoCheckBucket             bool                 `config:"no_check_bucket"`
+	Decompress                bool                 `config:"decompress"`
+	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote storage server
 type Fs struct {
-	name          string           // name of this remote
-	root          string           // the path we are working on if any
-	opt           Options          // parsed options
-	features      *fs.Features     // optional features
-	svc           *storage.Service // the connection to the storage server
-	client        *http.Client     // authorized client
-	rootBucket    string           // bucket part of root (if any)
-	rootDirectory string           // directory part of root (if any)
-	cache         *bucket.Cache    // cache of bucket status
-	pacer         *fs.Pacer        // To pace the API calls
+	name           string           // name of this remote
+	root           string           // the path we are working on if any
+	opt            Options          // parsed options
+	features       *fs.Features     // optional features
+	svc            *storage.Service // the connection to the storage server
+	client         *http.Client     // authorized client
+	rootBucket     string           // bucket part of root (if any)
+	rootDirectory  string           // directory part of root (if any)
+	cache          *bucket.Cache    // cache of bucket status
+	pacer          *fs.Pacer        // To pace the API calls
+	warnCompressed sync.Once        // warn once about compressed files
 }
 
 // Object describes a storage object
@@ -355,6 +377,7 @@ type Object struct {
 	bytes    int64     // Bytes in the object
 	modTime  time.Time // Modified time of the object
 	mimeType string
+	gzipped  bool // set if object has Content-Encoding: gzip
 }
 
 // ------------------------------------------------------------
@@ -372,7 +395,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootBucket == "" {
-		return fmt.Sprintf("GCS root")
+		return "GCS root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("GCS bucket %s", f.rootBucket)
@@ -464,7 +487,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -505,7 +528,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
-	f.svc, err = storage.New(f.client)
+	gcsOpts := []option.ClientOption{option.WithHTTPClient(f.client)}
+	if opt.Endpoint != "" {
+		gcsOpts = append(gcsOpts, option.WithEndpoint(opt.Endpoint))
+	}
+	f.svc, err = storage.NewService(context.Background(), gcsOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create Google Cloud Storage client: %w", err)
 	}
@@ -562,7 +589,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 //
 // dir is the starting directory, "" for root
 //
-// Set recurse to read sub directories
+// Set recurse to read sub directories.
 //
 // The remote has prefix removed from it and if addBucket is set
 // then it adds the bucket to the start.
@@ -780,7 +807,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -882,9 +909,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -975,6 +1002,7 @@ func (o *Object) setMetaData(info *storage.Object) {
 	o.url = info.MediaLink
 	o.bytes = int64(info.Size)
 	o.mimeType = info.ContentType
+	o.gzipped = info.ContentEncoding == "gzip"
 
 	// Read md5sum
 	md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@@ -1013,6 +1041,12 @@ func (o *Object) setMetaData(info *storage.Object) {
 	} else {
 		o.modTime = modTime
 	}
+
+	// If gunzipping then size and md5sum are unknown
+	if o.gzipped && o.fs.opt.Decompress {
+		o.bytes = -1
+		o.md5sum = ""
+	}
 }
 
 // readObjectInfo reads the definition for an object
@@ -1113,6 +1147,18 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 	fs.FixRangeOption(options, o.bytes)
+	if o.gzipped && !o.fs.opt.Decompress {
+		// Allow files which are stored on the cloud storage system
+		// compressed to be downloaded without being decompressed.  Note
+		// that setting this here overrides the automatic decompression
+		// in the Transport.
+		//
+		// See: https://cloud.google.com/storage/docs/transcoding
+		req.Header.Set("Accept-Encoding", "gzip")
+		o.fs.warnCompressed.Do(func() {
+			fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-decompress to override")
+		})
+	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/googlephotos/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Google Photos API.
 package api
 
 import (

backend/googlephotos/googlephotos.go

@@ -178,7 +178,7 @@ type Fs struct {
 	opt        Options                // parsed options
 	features   *fs.Features           // optional features
 	unAuth     *rest.Client           // unauthenticated http client
-	srv        *rest.Client           // the connection to the one drive server
+	srv        *rest.Client           // the connection to the server
 	ts         *oauthutil.TokenSource // token source for oauth2
 	pacer      *fs.Pacer              // To pace the API calls
 	startTime  time.Time              // time Fs was started - used for datestamps
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
 		for i := range items {
 			item := &result.MediaItems[i]
 			remote := item.Filename
-			remote = strings.Replace(remote, "/", "／", -1)
+			remote = strings.ReplaceAll(remote, "/", "／")
 			err = fn(remote, item, false)
 			if err != nil {
 				return err
@@ -661,7 +661,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {

backend/googlephotos/googlephotos_test.go

@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"testing"
@@ -37,7 +37,7 @@ func TestIntegration(t *testing.T) {
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
 	if err == fs.ErrorNotFoundInConfigFile {
-		t.Skip(fmt.Sprintf("Couldn't create google photos backend - skipping tests: %v", err))
+		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)
 
@@ -99,7 +99,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := ioutil.ReadAll(in)
+				buf, err := io.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)

backend/googlephotos/pattern.go

@@ -315,7 +315,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 
 // featureFilter creates a filter for the Feature enum
 //
-// The API only supports one feature, FAVORITES, so hardcode that feature
+// The API only supports one feature, FAVORITES, so hardcode that feature.
 //
 // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
 func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {

backend/googlephotos/pattern_test.go

@@ -50,7 +50,7 @@ func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums,
 
 // mock listUploads for testing
 func (f *testLister) listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entries, _ = f.uploaded[dir]
+	entries = f.uploaded[dir]
 	return entries, nil
 }
 

backend/hasher/hasher.go

@@ -27,6 +27,9 @@ func init() {
 		Name:        "hasher",
 		Description: "Better checksums for other remotes",
 		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
@@ -158,6 +161,11 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 		IsLocal:                 true,
 		ReadMimeType:            true,
 		WriteMimeType:           true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -282,7 +290,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	if do := f.Fs.Features().CleanUp; do != nil {
 		return do(ctx)
 	}
-	return errors.New("CleanUp not supported")
+	return errors.New("not supported by underlying remote")
 }
 
 // About gets quota information from the Fs
@@ -290,7 +298,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	if do := f.Fs.Features().About; do != nil {
 		return do(ctx)
 	}
-	return nil, errors.New("About not supported")
+	return nil, errors.New("not supported by underlying remote")
 }
 
 // ChangeNotify calls the passed function with a path that has had changes.
@@ -485,6 +493,17 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return ""
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
+	do, ok := o.Object.(fs.Metadataer)
+	if !ok {
+		return nil, nil
+	}
+	return do.Metadata(ctx)
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -507,10 +526,5 @@ var (
 	_ fs.UserInfoer      = (*Fs)(nil)
 	_ fs.Disconnecter    = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
-	_ fs.Object          = (*Object)(nil)
-	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
-	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.FullObject      = (*Object)(nil)
 )

backend/hasher/hasher_internal_test.go

@@ -19,7 +19,7 @@ import (
 func putFile(ctx context.Context, t *testing.T, f fs.Fs, name, data string) fs.Object {
 	mtime1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
 	item := fstest.Item{Path: name, ModTime: mtime1}
-	_, o := fstests.PutTestContents(ctx, t, f, &item, data, true)
+	o := fstests.PutTestContents(ctx, t, f, &item, data, true)
 	require.NotNil(t, o)
 	return o
 }
@@ -35,7 +35,7 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
 	// make a temporary crypt remote
 	ctx := context.Background()
 	pass := obscure.MustObscure("crypt")
-	remote := fmt.Sprintf(":crypt,remote=%s,password=%s:", tempRoot, pass)
+	remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass)
 	cryptFs, err := fs.NewFs(ctx, remote)
 	require.NoError(t, err)
 

backend/hasher/hasher_test.go

@@ -33,6 +33,7 @@ func TestIntegration(t *testing.T) {
 			{Name: "TestHasher", Key: "remote", Value: tempDir},
 		}
 		opt.RemoteName = "TestHasher:"
+		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
 }

backend/hasher/object.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"time"
 
@@ -118,7 +117,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(ioutil.Discard, r); err != nil {
+	if _, err = io.Copy(io.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}

backend/hdfs/fs.go

@@ -92,7 +92,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.ServicePrincipalName != "" {
 		options.KerberosClient, err = getKerberosClient()
 		if err != nil {
-			return nil, fmt.Errorf("Problem with kerberos authentication: %s", err)
+			return nil, fmt.Errorf("problem with kerberos authentication: %w", err)
 		}
 		options.KerberosServicePrincipleName = opt.ServicePrincipalName
 
@@ -265,9 +265,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/hdfs/hdfs.go

@@ -1,6 +1,7 @@
 //go:build !plan9
 // +build !plan9
 
+// Package hdfs provides an interface to the HDFS storage system.
 package hdfs
 
 import (

backend/hdfs/object.go

@@ -115,7 +115,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	info, err := o.fs.client.Stat(realpath)
+	_, err = o.fs.client.Stat(realpath)
 	if err == nil {
 		err = o.fs.client.Remove(realpath)
 		if err != nil {
@@ -147,7 +147,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	info, err = o.fs.client.Stat(realpath)
+	info, err := o.fs.client.Stat(realpath)
 	if err != nil {
 		return err
 	}

backend/hidrive/api/queries.go

@@ -0,0 +1,81 @@
+package api
+
+import (
+	"encoding/json"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+// Some presets for different amounts of information that can be requested for fields;
+// it is recommended to only request the information that is actually needed.
+var (
+	HiDriveObjectNoMetadataFields            = []string{"name", "type"}
+	HiDriveObjectWithMetadataFields          = append(HiDriveObjectNoMetadataFields, "id", "size", "mtime", "chash")
+	HiDriveObjectWithDirectoryMetadataFields = append(HiDriveObjectWithMetadataFields, "nmembers")
+	DirectoryContentFields                   = []string{"nmembers"}
+)
+
+// QueryParameters represents the parameters passed to an API-call.
+type QueryParameters struct {
+	url.Values
+}
+
+// NewQueryParameters initializes an instance of QueryParameters and
+// returns a pointer to it.
+func NewQueryParameters() *QueryParameters {
+	return &QueryParameters{url.Values{}}
+}
+
+// SetFileInDirectory sets the appropriate parameters
+// to specify a path to a file in a directory.
+// This is used by requests that work with paths for files that do not exist yet.
+// (For example when creating a file).
+// Most requests use the format produced by SetPath(...).
+func (p *QueryParameters) SetFileInDirectory(filePath string) {
+	directory, file := path.Split(path.Clean(filePath))
+	p.Set("dir", path.Clean(directory))
+	p.Set("name", file)
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetPath sets the appropriate parameters to access the given path.
+func (p *QueryParameters) SetPath(objectPath string) {
+	p.Set("path", path.Clean(objectPath))
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetTime sets the key to the time-value. It replaces any existing values.
+func (p *QueryParameters) SetTime(key string, value time.Time) error {
+	valueAPI := Time(value)
+	valueBytes, err := json.Marshal(&valueAPI)
+	if err != nil {
+		return err
+	}
+	p.Set(key, string(valueBytes))
+	return nil
+}
+
+// AddList adds the given values as a list
+// with each value separated by the separator.
+// It appends to any existing values associated with key.
+func (p *QueryParameters) AddList(key string, separator string, values ...string) {
+	original := p.Get(key)
+	p.Set(key, strings.Join(values, separator))
+	if original != "" {
+		p.Set(key, original+separator+p.Get(key))
+	}
+}
+
+// AddFields sets the appropriate parameter to access the given fields.
+// The given fields will be appended to any other existing fields.
+func (p *QueryParameters) AddFields(prefix string, fields ...string) {
+	modifiedFields := make([]string, len(fields))
+	for i, field := range fields {
+		modifiedFields[i] = prefix + field
+	}
+	p.AddList("fields", ",", modifiedFields...)
+}

backend/hidrive/api/types.go

@@ -0,0 +1,135 @@
+// Package api has type definitions and code related to API-calls for the HiDrive-API.
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// Time represents date and time information for the API.
+type Time time.Time
+
+// MarshalJSON turns Time into JSON (in Unix-time/UTC).
+func (t *Time) MarshalJSON() ([]byte, error) {
+	secs := time.Time(*t).Unix()
+	return []byte(strconv.FormatInt(secs, 10)), nil
+}
+
+// UnmarshalJSON turns JSON into Time.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	secs, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = Time(time.Unix(secs, 0))
+	return nil
+}
+
+// Error is returned from the API when things go wrong.
+type Error struct {
+	Code        json.Number `json:"code"`
+	ContextInfo json.RawMessage
+	Message     string `json:"msg"`
+}
+
+// Error returns a string for the error and satisfies the error interface.
+func (e *Error) Error() string {
+	out := fmt.Sprintf("Error %q", e.Code.String())
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.ContextInfo != nil {
+		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
+	}
+	return out
+}
+
+// Check Error satisfies the error interface.
+var _ error = (*Error)(nil)
+
+// possible types for HiDriveObject
+const (
+	HiDriveObjectTypeDirectory = "dir"
+	HiDriveObjectTypeFile      = "file"
+	HiDriveObjectTypeSymlink   = "symlink"
+)
+
+// HiDriveObject describes a folder, a symlink or a file.
+// Depending on the type and content, not all fields are present.
+type HiDriveObject struct {
+	Type         string `json:"type"`
+	ID           string `json:"id"`
+	ParentID     string `json:"parent_id"`
+	Name         string `json:"name"`
+	Path         string `json:"path"`
+	Size         int64  `json:"size"`
+	MemberCount  int64  `json:"nmembers"`
+	ModifiedAt   Time   `json:"mtime"`
+	ChangedAt    Time   `json:"ctime"`
+	MetaHash     string `json:"mhash"`
+	MetaOnlyHash string `json:"mohash"`
+	NameHash     string `json:"nhash"`
+	ContentHash  string `json:"chash"`
+	IsTeamfolder bool   `json:"teamfolder"`
+	Readable     bool   `json:"readable"`
+	Writable     bool   `json:"writable"`
+	Shareable    bool   `json:"shareable"`
+	MIMEType     string `json:"mime_type"`
+}
+
+// ModTime returns the modification time of the HiDriveObject.
+func (i *HiDriveObject) ModTime() time.Time {
+	t := time.Time(i.ModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.ChangedAt)
+	}
+	return t
+}
+
+// UnmarshalJSON turns JSON into HiDriveObject and
+// introduces specific default-values where necessary.
+func (i *HiDriveObject) UnmarshalJSON(data []byte) error {
+	type objectAlias HiDriveObject
+	defaultObject := objectAlias{
+		Size:        -1,
+		MemberCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultObject)
+	if err != nil {
+		return err
+	}
+	name, err := url.PathUnescape(defaultObject.Name)
+	if err == nil {
+		defaultObject.Name = name
+	}
+
+	*i = HiDriveObject(defaultObject)
+	return nil
+}
+
+// DirectoryContent describes the content of a directory.
+type DirectoryContent struct {
+	TotalCount int64           `json:"nmembers"`
+	Entries    []HiDriveObject `json:"members"`
+}
+
+// UnmarshalJSON turns JSON into DirectoryContent and
+// introduces specific default-values where necessary.
+func (d *DirectoryContent) UnmarshalJSON(data []byte) error {
+	type directoryContentAlias DirectoryContent
+	defaultDirectoryContent := directoryContentAlias{
+		TotalCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultDirectoryContent)
+	if err != nil {
+		return err
+	}
+
+	*d = DirectoryContent(defaultDirectoryContent)
+	return nil
+}

backend/hidrive/helpers.go

@@ -0,0 +1,888 @@
+package hidrive
+
+// This file is for helper-functions which may provide more general and
+// specialized functionality than the generic interfaces.
+// There are two sections:
+// 1. methods bound to Fs
+// 2. other functions independent from Fs used throughout the package
+
+// NOTE: Functions accessing paths expect any relative paths
+// to be resolved prior to execution with resolvePath(...).
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/hidrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/ranges"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+)
+
+const (
+	// MaximumUploadBytes represents the maximum amount of bytes
+	// a single upload-operation will support.
+	MaximumUploadBytes = 2147483647 // = 2GiB - 1
+	// iterationChunkSize represents the chunk size used to iterate directory contents.
+	iterationChunkSize = 5000
+)
+
+var (
+	// retryErrorCodes is a slice of error codes that we will always retry.
+	retryErrorCodes = []int{
+		429, // Too Many Requests
+		500, // Internal Server Error
+		502, // Bad Gateway
+		503, // Service Unavailable
+		504, // Gateway Timeout
+		509, // Bandwidth Limit Exceeded
+	}
+	// ErrorFileExists is returned when a query tries to create a file
+	// that already exists.
+	ErrorFileExists = errors.New("destination file already exists")
+)
+
+// MemberType represents the possible types of entries a directory can contain.
+type MemberType string
+
+// possible values for MemberType
+const (
+	AllMembers       MemberType = "all"
+	NoMembers        MemberType = "none"
+	DirectoryMembers MemberType = api.HiDriveObjectTypeDirectory
+	FileMembers      MemberType = api.HiDriveObjectTypeFile
+	SymlinkMembers   MemberType = api.HiDriveObjectTypeSymlink
+)
+
+// SortByField represents possible fields to sort entries of a directory by.
+type SortByField string
+
+// possible values for SortByField
+const (
+	descendingSort             string      = "-"
+	SortByName                 SortByField = "name"
+	SortByModTime              SortByField = "mtime"
+	SortByObjectType           SortByField = "type"
+	SortBySize                 SortByField = "size"
+	SortByNameDescending       SortByField = SortByField(descendingSort) + SortByName
+	SortByModTimeDescending    SortByField = SortByField(descendingSort) + SortByModTime
+	SortByObjectTypeDescending SortByField = SortByField(descendingSort) + SortByObjectType
+	SortBySizeDescending       SortByField = SortByField(descendingSort) + SortBySize
+)
+
+var (
+	// Unsorted disables sorting and can therefore not be combined with other values.
+	Unsorted = []SortByField{"none"}
+	// DefaultSorted does not specify how to sort and
+	// therefore implies the default sort order.
+	DefaultSorted = []SortByField{}
+)
+
+// CopyOrMoveOperationType represents the possible types of copy- and move-operations.
+type CopyOrMoveOperationType int
+
+// possible values for CopyOrMoveOperationType
+const (
+	MoveOriginal CopyOrMoveOperationType = iota
+	CopyOriginal
+	CopyOriginalPreserveModTime
+)
+
+// OnExistAction represents possible actions the API should take,
+// when a request tries to create a path that already exists.
+type OnExistAction string
+
+// possible values for OnExistAction
+const (
+	// IgnoreOnExist instructs the API not to execute
+	// the request in case of a conflict, but to return an error.
+	IgnoreOnExist OnExistAction = "ignore"
+	// AutoNameOnExist instructs the API to automatically rename
+	// any conflicting request-objects.
+	AutoNameOnExist OnExistAction = "autoname"
+	// OverwriteOnExist instructs the API to overwrite any conflicting files.
+	// This can only be used, if the request operates on files directly.
+	// (For example when moving/copying a file.)
+	// For most requests this action will simply be ignored.
+	OverwriteOnExist OnExistAction = "overwrite"
+)
+
+// shouldRetry returns a boolean as to whether this resp and err deserve to be retried.
+// It tries to expire/invalidate the token, if necessary.
+// It returns the err as a convenience.
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	if resp != nil && (resp.StatusCode == 401 || isHTTPError(err, 401)) && len(resp.Header["Www-Authenticate"]) > 0 {
+		fs.Debugf(f, "Token might be invalid: %v", err)
+		if f.tokenRenewer != nil {
+			iErr := f.tokenRenewer.Expire()
+			if iErr == nil {
+				return true, err
+			}
+		}
+	}
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// resolvePath resolves the given (relative) path and
+// returns a path suitable for API-calls.
+// This will consider the root-path of the fs and any needed prefixes.
+//
+// Any relative paths passed to functions that access these paths should
+// be resolved with this first!
+func (f *Fs) resolvePath(objectPath string) string {
+	resolved := path.Join(f.opt.RootPrefix, f.root, f.opt.Enc.FromStandardPath(objectPath))
+	return resolved
+}
+
+// iterateOverDirectory calls the given function callback
+// on each item found in a given directory.
+//
+// If callback ever returns true then this exits early with found = true.
+func (f *Fs) iterateOverDirectory(ctx context.Context, directory string, searchOnly MemberType, callback func(*api.HiDriveObject) bool, fields []string, sortBy []SortByField) (found bool, err error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.AddFields("members.", fields...)
+	parameters.AddFields("", api.DirectoryContentFields...)
+	parameters.Set("members", string(searchOnly))
+	for _, v := range sortBy {
+		// The explicit conversion is necessary for each element.
+		parameters.AddList("sort", ",", string(v))
+	}
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	iterateContent := func(result *api.DirectoryContent, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+		for _, item := range result.Entries {
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
+			if callback(&item) {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+	return f.paginateDirectoryAccess(ctx, &opts, iterationChunkSize, 0, iterateContent)
+}
+
+// paginateDirectoryAccess executes requests specified via ctx and opts
+// which should produce api.DirectoryContent.
+// This will paginate the requests using limit starting at the given offset.
+//
+// The given function callback is called on each api.DirectoryContent found
+// along with any errors that occurred.
+// If callback ever returns true then this exits early with found = true.
+// If callback ever returns an error then this exits early with that error.
+func (f *Fs) paginateDirectoryAccess(ctx context.Context, opts *rest.Opts, limit int64, offset int64, callback func(*api.DirectoryContent, error) (bool, error)) (found bool, err error) {
+	for {
+		opts.Parameters.Set("limit", strconv.FormatInt(offset, 10)+","+strconv.FormatInt(limit, 10))
+
+		var result api.DirectoryContent
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, opts, nil, &result)
+			return f.shouldRetry(ctx, resp, err)
+		})
+
+		found, err = callback(&result, err)
+		if found || err != nil {
+			return found, err
+		}
+
+		offset += int64(len(result.Entries))
+		if offset >= result.TotalCount || limit > int64(len(result.Entries)) {
+			break
+		}
+	}
+	return false, nil
+}
+
+// fetchMetadataForPath reads the metadata from the path.
+func (f *Fs) fetchMetadataForPath(ctx context.Context, path string, fields []string) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.AddFields("", fields...)
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/meta",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// copyOrMove copies or moves a directory or file
+// from the source-path to the destination-path.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: Use the explicit methods instead of directly invoking this method.
+// (Those are: copyDirectory, moveDirectory, copyFile, moveFile.)
+func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType CopyOrMoveOperationType, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.Set("src", source)
+	parameters.Set("dst", destination)
+	if onExist == AutoNameOnExist ||
+		(onExist == OverwriteOnExist && !isDirectory) {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	endpoint := "/"
+	if isDirectory {
+		endpoint += "dir"
+	} else {
+		endpoint += "file"
+	}
+	switch operationType {
+	case MoveOriginal:
+		endpoint += "/move"
+	case CopyOriginalPreserveModTime:
+		parameters.Set("preserve_mtime", strconv.FormatBool(true))
+		fallthrough
+	case CopyOriginal:
+		endpoint += "/copy"
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       endpoint,
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// copyDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) copyDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
+// moveDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) moveDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, MoveOriginal, source, destination, onExist)
+}
+
+// copyFile copies the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation will expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) copyFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
+// moveFile moves the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation may expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) moveFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, MoveOriginal, source, destination, onExist)
+}
+
+// createDirectory creates the directory at the given path and
+// returns the resulting api-object if successful.
+//
+// The directory will only be created if its parent-directory exists.
+// This returns fs.ErrorDirNotFound if the parent-directory is not found.
+// This returns fs.ErrorDirExists if the directory already exists.
+func (f *Fs) createDirectory(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, fs.ErrorDirExists
+	}
+	return nil, err
+}
+
+// createDirectories creates the directory at the given path
+// along with any missing parent directories and
+// returns the resulting api-object (of the created directory) if successful.
+//
+// This returns fs.ErrorDirExists if the directory already exists.
+//
+// If an error occurs while the parent directories are being created,
+// any directories already created will NOT be deleted again.
+func (f *Fs) createDirectories(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	result, err := f.createDirectory(ctx, directory, onExist)
+	if err == nil {
+		return result, nil
+	}
+	if err != fs.ErrorDirNotFound {
+		return nil, err
+	}
+	parentDirectory := path.Dir(directory)
+	_, err = f.createDirectories(ctx, parentDirectory, onExist)
+	if err != nil && err != fs.ErrorDirExists {
+		return nil, err
+	}
+	// NOTE: Ignoring fs.ErrorDirExists does no harm,
+	// since it does not mean the child directory cannot be created.
+	return f.createDirectory(ctx, directory, onExist)
+}
+
+// deleteDirectory deletes the directory at the given path.
+//
+// If recursive is false, the directory will only be deleted if it is empty.
+// If recursive is true, the directory will be deleted regardless of its content.
+// This returns fs.ErrorDirNotFound if the directory is not found.
+// This returns fs.ErrorDirectoryNotEmpty if the directory is not empty and
+// recursive is false.
+func (f *Fs) deleteDirectory(ctx context.Context, directory string, recursive bool) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.Set("recursive", strconv.FormatBool(recursive))
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case isHTTPError(err, 404):
+		return fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return fs.ErrorDirectoryNotEmpty
+	}
+	return err
+}
+
+// deleteObject deletes the object/file at the given path.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) deleteObject(ctx context.Context, path string) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/file",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// createFile creates a file at the given path
+// with the content of the io.ReadSeeker.
+// This guarantees that existing files will not be overwritten.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+// This returns ErrorFileExists if a file already exists at the specified path.
+func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "POST",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, ErrorFileExists
+	}
+	return nil, err
+}
+
+// overwriteFile updates the content of the file at the given path
+// with the content of the io.ReadSeeker.
+// If the file does not exist it will be created.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PUT",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	}
+	return nil, err
+}
+
+// uploadFileChunked updates the content of the existing file at the given path
+// with the content of the io.Reader.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+// Returns the resulting api-object if successful.
+//
+// Replaces the file contents by uploading multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: This method uses updateFileChunked and may create sparse files,
+// if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) uploadFileChunked(ctx context.Context, path string, content io.Reader, modTime time.Time, chunkSize int, transferLimit int64) (okSize uint64, info *api.HiDriveObject, err error) {
+	okSize, err = f.updateFileChunked(ctx, path, content, 0, chunkSize, transferLimit)
+
+	if err == nil {
+		info, err = f.resizeFile(ctx, path, okSize, modTime)
+	}
+	return okSize, info, err
+}
+
+// updateFileChunked updates the content of the existing file at the given path
+// starting at the given offset.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.Reader.
+// If the offset is beyond the file end, the file is extended up to the offset.
+//
+// The upload is done multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+//
+// NOTE: Because it is inefficient to set the modification time with every chunk,
+// setting it to a specific value must be done in a separate request
+// after this operation finishes.
+//
+// NOTE: This method uses patchFile and may create sparse files,
+// especially if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Reader, offset uint64, chunkSize int, transferLimit int64) (okSize uint64, err error) {
+	var (
+		okChunksMu sync.Mutex // protects the variables below
+		okChunks   []ranges.Range
+	)
+	g, gCtx := errgroup.WithContext(ctx)
+	transferSemaphore := semaphore.NewWeighted(transferLimit)
+
+	var readErr error
+	startMoreTransfers := true
+	zeroTime := time.Time{}
+	for chunk := uint64(0); startMoreTransfers; chunk++ {
+		// Acquire semaphore to limit number of transfers in parallel.
+		readErr = transferSemaphore.Acquire(gCtx, 1)
+		if readErr != nil {
+			break
+		}
+
+		// Read a chunk of data.
+		chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
+		if bytesRead < chunkSize {
+			startMoreTransfers = false
+		}
+		if readErr != nil || bytesRead <= 0 {
+			break
+		}
+
+		// Transfer the chunk.
+		chunkOffset := uint64(chunkSize)*chunk + offset
+		g.Go(func() error {
+			// After this upload is done,
+			// signal that another transfer can be started.
+			defer transferSemaphore.Release(1)
+			uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
+			if uploadErr == nil {
+				// Remember successfully written chunks.
+				okChunksMu.Lock()
+				okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
+				okChunksMu.Unlock()
+				fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
+			} else {
+				fs.Infof(f, "Error while uploading chunk at offset %v. Error is %v.", chunkOffset, uploadErr)
+			}
+			return uploadErr
+		})
+	}
+
+	if readErr != nil {
+		// Log the error in case it is later ignored because of an upload-error.
+		fs.Infof(f, "Error while reading/preparing to upload a chunk. Error is %v.", readErr)
+	}
+
+	err = g.Wait()
+
+	// Compute the first continuous range of the file content,
+	// which does not contain any failed chunks.
+	// Do not forget to add the file content up to the starting offset,
+	// which is presumed to be already correct.
+	rs := ranges.Ranges{}
+	rs.Insert(ranges.Range{Pos: 0, Size: int64(offset)})
+	for _, chunkRange := range okChunks {
+		rs.Insert(chunkRange)
+	}
+	if len(rs) > 0 && rs[0].Pos == 0 {
+		okSize = uint64(rs[0].Size)
+	}
+
+	if err != nil {
+		return okSize, err
+	}
+	if readErr != nil {
+		return okSize, readErr
+	}
+
+	return okSize, nil
+}
+
+// patchFile updates the content of the existing file at the given path
+// starting at the given offset.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.ReadSeeker.
+// If the offset is beyond the file end, the file is extended up to the offset.
+// The maximum size of the update is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file up to the offset this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("offset", strconv.FormatUint(offset, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PATCH",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+		NoResponse:  true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		_, err = content.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		resp, err = f.srv.Call(ctx, &opts)
+		if isHTTPError(err, 423) {
+			return true, err
+		}
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// resizeFile updates the existing file at the given path to be of the given size
+// and returns the resulting api-object if successful.
+//
+// If the given size is smaller than the current filesize,
+// the file is cut/truncated at that position.
+// If the given size is larger, the file is extended up to that position.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) resizeFile(ctx context.Context, path string, size uint64, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("size", strconv.FormatUint(size, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/file/truncate",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorObjectNotFound
+	}
+	return nil, err
+}
+
+// ------------------------------------------------------------
+
+// isHTTPError compares the numerical status code
+// of an api.Error to the given HTTP status.
+//
+// If the given error is not an api.Error or
+// a numerical status code could not be determined, this returns false.
+// Otherwise this returns whether the status code of the error is equal to the given status.
+func isHTTPError(err error, status int64) bool {
+	if apiErr, ok := err.(*api.Error); ok {
+		errStatus, decodeErr := apiErr.Code.Int64()
+		if decodeErr == nil && errStatus == status {
+			return true
+		}
+	}
+	return false
+}
+
+// createHiDriveScopes creates oauth-scopes
+// from the given user-role and access-permissions.
+//
+// If the arguments are empty, they will not be included in the result.
+func createHiDriveScopes(role string, access string) []string {
+	switch {
+	case role != "" && access != "":
+		return []string{access + "," + role}
+	case role != "":
+		return []string{role}
+	case access != "":
+		return []string{access}
+	}
+	return []string{}
+}
+
+// cachedReader returns a version of the reader that caches its contents and
+// can therefore be reset using Seek.
+func cachedReader(reader io.Reader) io.ReadSeeker {
+	bytesReader, ok := reader.(*bytes.Reader)
+	if ok {
+		return bytesReader
+	}
+
+	repeatableReader, ok := reader.(*readers.RepeatableReader)
+	if ok {
+		return repeatableReader
+	}
+
+	return readers.NewRepeatableReader(reader)
+}
+
+// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
+// Returns a new io.Reader (chunkReader) for that chunk
+// and the number of bytes that have been read from reader.
+func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
+	// Unwrap any accounting from the input if present.
+	reader, wrap := accounting.UnWrap(reader)
+
+	// Read a chunk of data.
+	buffer := make([]byte, length)
+	bytesRead, err = io.ReadFull(reader, buffer)
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
+		err = nil
+	}
+	if err != nil {
+		return nil, bytesRead, err
+	}
+	// Truncate unused capacity.
+	buffer = buffer[:bytesRead]
+
+	// Use wrap to put any accounting back for chunkReader.
+	return wrap(bytes.NewReader(buffer)), bytesRead, nil
+}

backend/hidrive/hidrive_test.go

@@ -0,0 +1,45 @@
+// Test HiDrive filesystem interface
+package hidrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote.
+func TestIntegration(t *testing.T) {
+	name := "TestHiDrive"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize:       1,
+			MaxChunkSize:       MaximumUploadBytes,
+			CeilChunkSize:      nil,
+			NeedMultipleChunks: false,
+		},
+	})
+}
+
+// Change the configured UploadChunkSize.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadChunkSize(chunksize fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadChunkSize = f.opt.UploadChunkSize, chunksize
+	return old, nil
+}
+
+// Change the configured UploadCutoff.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
+	return old, nil
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)

backend/hidrive/hidrivehash/hidrivehash.go

@@ -0,0 +1,410 @@
+// Package hidrivehash implements the HiDrive hashing algorithm which combines SHA-1 hashes hierarchically to a single top-level hash.
+//
+// Note: This implementation does not grant access to any partial hashes generated.
+//
+// See: https://developer.hidrive.com/wp-content/uploads/2021/07/HiDrive_Synchronization-v3.3-rev28.pdf
+// (link to newest version: https://static.hidrive.com/dev/0001)
+package hidrivehash
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+)
+
+const (
+	// BlockSize of the checksum in bytes.
+	BlockSize = 4096
+	// Size of the checksum in bytes.
+	Size = sha1.Size
+	// sumsPerLevel is the number of checksums
+	sumsPerLevel = 256
+)
+
+var (
+	// zeroSum is a special hash consisting of 20 null-bytes.
+	// This will be the hash of any empty file (or ones containing only null-bytes).
+	zeroSum = [Size]byte{}
+	// ErrorInvalidEncoding is returned when a hash should be decoded from a binary form that is invalid.
+	ErrorInvalidEncoding = errors.New("encoded binary form is invalid for this hash")
+	// ErrorHashFull is returned when a hash reached its capacity and cannot accept any more input.
+	ErrorHashFull = errors.New("hash reached its capacity")
+)
+
+// writeByBlock writes len(p) bytes from p to the io.Writer in blocks of size blockSize.
+// It returns the number of bytes written from p (0 <= n <= len(p))
+// and any error encountered that caused the write to stop early.
+//
+// A pointer bytesInBlock to a counter needs to be supplied,
+// that is used to keep track how many bytes have been written to the writer already.
+// A pointer onlyNullBytesInBlock to a boolean needs to be supplied,
+// that is used to keep track whether the block so far only consists of null-bytes.
+// The callback onBlockWritten is called whenever a full block has been written to the writer
+// and is given as input the number of bytes that still need to be written.
+func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *uint32, onlyNullBytesInBlock *bool, onBlockWritten func(remaining int) error) (n int, err error) {
+	total := len(p)
+	nullBytes := make([]byte, blockSize)
+	for len(p) > 0 {
+		toWrite := int(blockSize - *bytesInBlock)
+		if toWrite > len(p) {
+			toWrite = len(p)
+		}
+		c, err := writer.Write(p[:toWrite])
+		*bytesInBlock += uint32(c)
+		*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
+		// Discard data written through a reslice
+		p = p[c:]
+		if err != nil {
+			return total - len(p), err
+		}
+		if *bytesInBlock == blockSize {
+			err = onBlockWritten(len(p))
+			if err != nil {
+				return total - len(p), err
+			}
+			*bytesInBlock = 0
+			*onlyNullBytesInBlock = true
+		}
+	}
+	return total, nil
+}
+
+// level is a hash.Hash that is used to aggregate the checksums produced by the level hierarchically beneath it.
+// It is used to represent any level-n hash, except for level-0.
+type level struct {
+	checksum              [Size]byte // aggregated checksum of this level
+	sumCount              uint32     // number of sums contained in this level so far
+	bytesInHasher         uint32     //  number of bytes written into hasher so far
+	onlyNullBytesInHasher bool       // whether the hasher only contains null-bytes so far
+	hasher                hash.Hash
+}
+
+// NewLevel returns a new hash.Hash computing any level-n hash, except level-0.
+func NewLevel() hash.Hash {
+	l := &level{}
+	l.Reset()
+	return l
+}
+
+// Add takes a position-embedded SHA-1 checksum and adds it to the level.
+func (l *level) Add(sha1sum []byte) {
+	var tmp uint
+	var carry bool
+	for i := Size - 1; i >= 0; i-- {
+		tmp = uint(sha1sum[i]) + uint(l.checksum[i])
+		if carry {
+			tmp++
+		}
+		carry = tmp > 255
+		l.checksum[i] = byte(tmp)
+	}
+}
+
+// IsFull returns whether the number of checksums added to this level reached its capacity.
+func (l *level) IsFull() bool {
+	return l.sumCount >= sumsPerLevel
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// Contrary to the specification from hash.Hash, this DOES return an error,
+// specifically ErrorHashFull if and only if IsFull() returns true.
+func (l *level) Write(p []byte) (n int, err error) {
+	if l.IsFull() {
+		return 0, ErrorHashFull
+	}
+	onBlockWritten := func(remaining int) error {
+		if !l.onlyNullBytesInHasher {
+			c, err := l.hasher.Write([]byte{byte(l.sumCount)})
+			l.bytesInHasher += uint32(c)
+			if err != nil {
+				return err
+			}
+			l.Add(l.hasher.Sum(nil))
+		}
+		l.sumCount++
+		l.hasher.Reset()
+		if remaining > 0 && l.IsFull() {
+			return ErrorHashFull
+		}
+		return nil
+	}
+	return writeByBlock(p, l.hasher, uint32(l.BlockSize()), &l.bytesInHasher, &l.onlyNullBytesInHasher, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (l *level) Sum(b []byte) []byte {
+	return append(b, l.checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (l *level) Reset() {
+	l.checksum = zeroSum // clear the current checksum
+	l.sumCount = 0
+	l.bytesInHasher = 0
+	l.onlyNullBytesInHasher = true
+	l.hasher = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (l *level) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (l *level) BlockSize() int {
+	return Size
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (l *level) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+4+1)
+	copy(b, l.checksum[:])
+	binary.BigEndian.PutUint32(b[Size:], l.sumCount)
+	binary.BigEndian.PutUint32(b[Size+4:], l.bytesInHasher)
+	if l.onlyNullBytesInHasher {
+		b[Size+4+4] = 1
+	}
+	encodedHasher, err := l.hasher.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedHasher...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (l *level) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+4+1 {
+		return ErrorInvalidEncoding
+	}
+	copy(l.checksum[:], b)
+	l.sumCount = binary.BigEndian.Uint32(b[Size:])
+	l.bytesInHasher = binary.BigEndian.Uint32(b[Size+4:])
+	switch b[Size+4+4] {
+	case 0:
+		l.onlyNullBytesInHasher = false
+	case 1:
+		l.onlyNullBytesInHasher = true
+	default:
+		return ErrorInvalidEncoding
+	}
+	err := l.hasher.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[Size+4+4+1:])
+	return err
+}
+
+// hidriveHash is the hash computing the actual checksum used by HiDrive by combining multiple level-hashes.
+type hidriveHash struct {
+	levels               []*level   // collection of level-hashes, one for each level starting at level-1
+	lastSumWritten       [Size]byte // the last checksum written to any of the levels
+	bytesInBlock         uint32     // bytes written into blockHash so far
+	onlyNullBytesInBlock bool       // whether the hasher only contains null-bytes so far
+	blockHash            hash.Hash
+}
+
+// New returns a new hash.Hash computing the HiDrive checksum.
+func New() hash.Hash {
+	h := &hidriveHash{}
+	h.Reset()
+	return h
+}
+
+// aggregateToLevel writes the checksum to the level at the given index
+// and if necessary propagates any changes to levels above.
+func (h *hidriveHash) aggregateToLevel(index int, sum []byte) {
+	for i := index; ; i++ {
+		if i >= len(h.levels) {
+			h.levels = append(h.levels, NewLevel().(*level))
+		}
+		_, err := h.levels[i].Write(sum)
+		copy(h.lastSumWritten[:], sum)
+		if err != nil {
+			panic(fmt.Errorf("level-hash should not have produced an error: %w", err))
+		}
+		if !h.levels[i].IsFull() {
+			break
+		}
+		sum = h.levels[i].Sum(nil)
+		h.levels[i].Reset()
+	}
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+func (h *hidriveHash) Write(p []byte) (n int, err error) {
+	onBlockWritten := func(remaining int) error {
+		var sum []byte
+		if h.onlyNullBytesInBlock {
+			sum = zeroSum[:]
+		} else {
+			sum = h.blockHash.Sum(nil)
+		}
+		h.blockHash.Reset()
+		h.aggregateToLevel(0, sum)
+		return nil
+	}
+	return writeByBlock(p, h.blockHash, uint32(BlockSize), &h.bytesInBlock, &h.onlyNullBytesInBlock, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (h *hidriveHash) Sum(b []byte) []byte {
+	// Save internal state.
+	state, err := h.MarshalBinary()
+	if err != nil {
+		panic(fmt.Errorf("saving the internal state should not have produced an error: %w", err))
+	}
+
+	if h.bytesInBlock > 0 {
+		// Fill remainder of block with null-bytes.
+		filler := make([]byte, h.BlockSize()-int(h.bytesInBlock))
+		_, err = h.Write(filler)
+		if err != nil {
+			panic(fmt.Errorf("filling with null-bytes should not have an error: %w", err))
+		}
+	}
+
+	checksum := zeroSum
+	for i := 0; i < len(h.levels); i++ {
+		level := h.levels[i]
+		if i < len(h.levels)-1 {
+			// Aggregate non-empty non-final levels.
+			if level.sumCount >= 1 {
+				h.aggregateToLevel(i+1, level.Sum(nil))
+				level.Reset()
+			}
+		} else {
+			// Determine sum of final level.
+			if level.sumCount > 1 {
+				copy(checksum[:], level.Sum(nil))
+			} else {
+				// This is needed, otherwise there is no way to return
+				// the non-position-embedded checksum.
+				checksum = h.lastSumWritten
+			}
+		}
+	}
+
+	// Restore internal state.
+	err = h.UnmarshalBinary(state)
+	if err != nil {
+		panic(fmt.Errorf("restoring the internal state should not have produced an error: %w", err))
+	}
+
+	return append(b, checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (h *hidriveHash) Reset() {
+	h.levels = nil
+	h.lastSumWritten = zeroSum // clear the last written checksum
+	h.bytesInBlock = 0
+	h.onlyNullBytesInBlock = true
+	h.blockHash = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (h *hidriveHash) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (h *hidriveHash) BlockSize() int {
+	return BlockSize
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (h *hidriveHash) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+1+8)
+	copy(b, h.lastSumWritten[:])
+	binary.BigEndian.PutUint32(b[Size:], h.bytesInBlock)
+	if h.onlyNullBytesInBlock {
+		b[Size+4] = 1
+	}
+
+	binary.BigEndian.PutUint64(b[Size+4+1:], uint64(len(h.levels)))
+	for _, level := range h.levels {
+		encodedLevel, err := level.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+		encodedLength := make([]byte, 8)
+		binary.BigEndian.PutUint64(encodedLength, uint64(len(encodedLevel)))
+		b = append(b, encodedLength...)
+		b = append(b, encodedLevel...)
+	}
+	encodedBlockHash, err := h.blockHash.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedBlockHash...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (h *hidriveHash) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+1+8 {
+		return ErrorInvalidEncoding
+	}
+	copy(h.lastSumWritten[:], b)
+	h.bytesInBlock = binary.BigEndian.Uint32(b[Size:])
+	switch b[Size+4] {
+	case 0:
+		h.onlyNullBytesInBlock = false
+	case 1:
+		h.onlyNullBytesInBlock = true
+	default:
+		return ErrorInvalidEncoding
+	}
+
+	amount := binary.BigEndian.Uint64(b[Size+4+1:])
+	h.levels = make([]*level, int(amount))
+	offset := Size + 4 + 1 + 8
+	for i := range h.levels {
+		length := int(binary.BigEndian.Uint64(b[offset:]))
+		offset += 8
+		h.levels[i] = NewLevel().(*level)
+		err := h.levels[i].UnmarshalBinary(b[offset : offset+length])
+		if err != nil {
+			return err
+		}
+		offset += length
+	}
+	err := h.blockHash.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[offset:])
+	return err
+}
+
+// Sum returns the HiDrive checksum of the data.
+func Sum(data []byte) [Size]byte {
+	h := New().(*hidriveHash)
+	_, _ = h.Write(data)
+	var result [Size]byte
+	copy(result[:], h.Sum(nil))
+	return result
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ hash.Hash                  = (*level)(nil)
+	_ encoding.BinaryMarshaler   = (*level)(nil)
+	_ encoding.BinaryUnmarshaler = (*level)(nil)
+	_ internal.LevelHash         = (*level)(nil)
+	_ hash.Hash                  = (*hidriveHash)(nil)
+	_ encoding.BinaryMarshaler   = (*hidriveHash)(nil)
+	_ encoding.BinaryUnmarshaler = (*hidriveHash)(nil)
+)

backend/hidrive/hidrivehash/hidrivehash_test.go

@@ -0,0 +1,395 @@
+package hidrivehash_test
+
+import (
+	"crypto/sha1"
+	"encoding"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash"
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+// helper functions to set up test-tables
+
+func sha1ArrayAsSlice(sum [sha1.Size]byte) []byte {
+	return sum[:]
+}
+
+func mustDecode(hexstring string) []byte {
+	result, err := hex.DecodeString(hexstring)
+	if err != nil {
+		panic(err)
+	}
+	return result
+}
+
+// ------------------------------------------------------------
+
+var testTableLevelPositionEmbedded = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{78, 188, 156, 219, 173, 54, 81, 55, 47, 220, 222, 207, 201, 21, 57, 252, 255, 239, 251, 186}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{68, 135, 96, 187, 38, 253, 14, 167, 186, 167, 188, 210, 91, 177, 185, 13, 208, 217, 94, 18}),
+		},
+		"documentation-v3.2rev27-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{75, 211, 153, 190, 125, 179, 67, 49, 60, 149, 98, 246, 142, 20, 11, 254, 159, 162, 129, 237}),
+			sha1ArrayAsSlice([20]byte{150, 2, 9, 153, 97, 153, 189, 104, 147, 14, 77, 203, 244, 243, 25, 212, 67, 48, 111, 107}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{144, 209, 246, 100, 177, 216, 171, 229, 83, 17, 92, 135, 68, 98, 76, 72, 217, 24, 99, 176}),
+			sha1ArrayAsSlice([20]byte{38, 211, 255, 254, 19, 114, 105, 77, 230, 31, 170, 83, 57, 85, 102, 29, 28, 72, 211, 27}),
+		},
+		"documentation-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{40, 34, 8, 238, 37, 5, 237, 184, 79, 105, 10, 167, 171, 254, 13, 229, 132, 112, 254, 8}),
+			sha1ArrayAsSlice([20]byte{39, 112, 26, 86, 190, 35, 100, 101, 28, 131, 122, 191, 254, 144, 239, 107, 253, 124, 104, 203}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{213, 157, 141, 227, 213, 178, 25, 111, 200, 145, 77, 164, 17, 247, 202, 167, 37, 46, 0, 124}),
+			sha1ArrayAsSlice([20]byte{253, 13, 168, 58, 147, 213, 125, 212, 229, 20, 200, 100, 16, 136, 186, 19, 34, 170, 105, 71}),
+		},
+		"documentation-example L1 (position-embedded)",
+	},
+}
+
+var testTableLevel = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		},
+		[][]byte{
+			mustDecode("44fe5ca6342568b4167bf990b64e404a3975e1c3"),
+			mustDecode("90d1f664b1d8abe553115c8744624c48d91863b0"),
+			mustDecode("26d3fffe1372694de61faa533955661d1c48d31b"),
+		},
+		"documentation-example L0",
+	},
+	{
+		[][]byte{
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+		},
+		[][]byte{
+			mustDecode("ad7b84f5b0ac2bb7792842fc65f9bcc1a0bd0274"),
+			mustDecode("d59d8de3d5b2196fc8914da411f7caa7252e007c"),
+			mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		},
+		"documentation-example L1",
+	},
+	{
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+			mustDecode("0000000000000000000000000000000000000000"),
+		},
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+		},
+		"mixed-with-empties",
+	},
+}
+
+var testTable = []struct {
+	data []byte
+	// pattern describes how to use data to construct the hash-input.
+	// For every entry n at even indices this repeats the data n times.
+	// For every entry m at odd indices this repeats a null-byte m times.
+	// The input-data is constructed by concatenating the results in order.
+	pattern []int64
+	out     []byte
+	name    string
+}{
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64},
+		mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		"documentation-example L0",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256},
+		mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+		"documentation-example L1",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256, 0, 64 * 128, 4096 * 128, 64*2 + 32},
+		mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		"documentation-example L2",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{316},
+		mustDecode("72370f9c18a2c20b31d71f3f4cee7a3cd2703737"),
+		"not-block-aligned",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{13, 4096 * 3, 4},
+		mustDecode("a6990b81791f0d2db750b38f046df321c975aa60"),
+		"not-block-aligned-with-null-bytes",
+	},
+	{
+		[]byte{},
+		[]int64{},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"empty",
+	},
+	{
+		[]byte{},
+		[]int64{0, 4096 * 256 * 256},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"null-bytes",
+	},
+}
+
+// ------------------------------------------------------------
+
+func TestLevelAdd(t *testing.T) {
+	for _, test := range testTableLevelPositionEmbedded {
+		l := hidrivehash.NewLevel().(internal.LevelHash)
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Add(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelWrite(t *testing.T) {
+	for _, test := range testTableLevel {
+		l := hidrivehash.NewLevel()
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Write(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelIsFull(t *testing.T) {
+	content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel()
+	for i := 0; i < 256; i++ {
+		assert.False(t, l.(internal.LevelHash).IsFull())
+		written, err := l.Write(content[:])
+		assert.Equal(t, len(content), written)
+		if !assert.NoError(t, err) {
+			t.FailNow()
+		}
+	}
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	written, err := l.Write(content[:])
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	assert.Equal(t, 0, written)
+	assert.ErrorIs(t, err, hidrivehash.ErrorHashFull)
+}
+
+func TestLevelReset(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	zeroHash := l.Sum(nil)
+	_, err := l.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, l.Sum(nil))
+		l.Reset()
+		assert.Equal(t, zeroHash, l.Sum(nil))
+	}
+}
+
+func TestLevelSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.Size())
+}
+
+func TestLevelBlockSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.BlockSize())
+}
+
+func TestLevelBinaryMarshaler(t *testing.T) {
+	content := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	l.Write(content[:10])
+	encoded, err := l.MarshalBinary()
+	if assert.NoError(t, err) {
+		d := hidrivehash.NewLevel().(internal.LevelHash)
+		err = d.UnmarshalBinary(encoded)
+		if assert.NoError(t, err) {
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+			l.Write(content[10:])
+			d.Write(content[10:])
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+		}
+	}
+}
+
+func TestLevelInvalidEncoding(t *testing.T) {
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	err := l.UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+// ------------------------------------------------------------
+
+type infiniteReader struct {
+	source []byte
+	offset int
+}
+
+func (m *infiniteReader) Read(b []byte) (int, error) {
+	count := copy(b, m.source[m.offset:])
+	m.offset += count
+	m.offset %= len(m.source)
+	return count, nil
+}
+
+func writeInChunks(writer io.Writer, chunkSize int64, data []byte, pattern []int64) error {
+	readers := make([]io.Reader, len(pattern))
+	nullBytes := [4096]byte{}
+	for i, n := range pattern {
+		if i%2 == 0 {
+			readers[i] = io.LimitReader(&infiniteReader{data, 0}, n*int64(len(data)))
+		} else {
+			readers[i] = io.LimitReader(&infiniteReader{nullBytes[:], 0}, n)
+		}
+	}
+	reader := io.MultiReader(readers...)
+	for {
+		_, err := io.CopyN(writer, reader, chunkSize)
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return err
+		}
+	}
+}
+
+func TestWrite(t *testing.T) {
+	for _, test := range testTable {
+		t.Run(test.name, func(t *testing.T) {
+			h := hidrivehash.New()
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern)
+			if assert.NoError(t, err) {
+				normalSum := h.Sum(nil)
+				assert.Equal(t, test.out, normalSum)
+				// Test if different block-sizes produce differing results.
+				for _, blockSize := range []int64{397, 512, 4091, 8192, 10000} {
+					t.Run(fmt.Sprintf("block-size %v", blockSize), func(t *testing.T) {
+						h := hidrivehash.New()
+						err := writeInChunks(h, blockSize, test.data, test.pattern)
+						if assert.NoError(t, err) {
+							assert.Equal(t, normalSum, h.Sum(nil))
+						}
+					})
+				}
+			}
+		})
+	}
+}
+
+func TestReset(t *testing.T) {
+	h := hidrivehash.New()
+	zeroHash := h.Sum(nil)
+	_, err := h.Write([]byte{1})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, h.Sum(nil))
+		h.Reset()
+		assert.Equal(t, zeroHash, h.Sum(nil))
+	}
+}
+
+func TestSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 20, h.Size())
+}
+
+func TestBlockSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 4096, h.BlockSize())
+}
+
+func TestBinaryMarshaler(t *testing.T) {
+	for _, test := range testTable {
+		h := hidrivehash.New()
+		d := hidrivehash.New()
+		half := len(test.pattern) / 2
+		t.Run(test.name, func(t *testing.T) {
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[:half])
+			assert.NoError(t, err)
+			encoded, err := h.(encoding.BinaryMarshaler).MarshalBinary()
+			if assert.NoError(t, err) {
+				err = d.(encoding.BinaryUnmarshaler).UnmarshalBinary(encoded)
+				if assert.NoError(t, err) {
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+					err = writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					err = writeInChunks(d, int64(d.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+				}
+			}
+		})
+	}
+}
+
+func TestInvalidEncoding(t *testing.T) {
+	h := hidrivehash.New()
+	err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+func TestSum(t *testing.T) {
+	assert.Equal(t, [hidrivehash.Size]byte{}, hidrivehash.Sum([]byte{}))
+	content := []byte{1}
+	h := hidrivehash.New()
+	h.Write(content)
+	sum := hidrivehash.Sum(content)
+	assert.Equal(t, h.Sum(nil), sum[:])
+}

backend/hidrive/hidrivehash/internal/internal.go

@@ -0,0 +1,18 @@
+// Package internal provides utilities for HiDrive.
+package internal
+
+import (
+	"encoding"
+	"hash"
+)
+
+// LevelHash is an internal interface for level-hashes.
+type LevelHash interface {
+	encoding.BinaryMarshaler
+	encoding.BinaryUnmarshaler
+	hash.Hash
+	// Add takes a position-embedded checksum and adds it to the level.
+	Add(sum []byte)
+	// IsFull returns whether the number of checksums added to this level reached its capacity.
+	IsFull() bool
+}

backend/http/http.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -35,11 +34,11 @@ var (
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "http",
-		Description: "http Connection",
+		Description: "HTTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "url",
-			Help:     "URL of http host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
+			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
 			Required: true,
 		}, {
 			Name: "headers",
@@ -305,7 +304,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat(ctx)
+	err := o.head(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -317,15 +316,6 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-// parse s into an int64, on failure return def
-func parseInt64(s string, def int64) int64 {
-	n, e := strconv.ParseInt(s, 10, 64)
-	if e != nil {
-		return def
-	}
-	return n
-}
-
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -500,7 +490,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.stat(ctx); err {
+				switch err := file.head(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
@@ -579,8 +569,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// stat updates the info field in the Object
-func (o *Object) stat(ctx context.Context) error {
+// head sends a HEAD request to update info fields in the Object
+func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -601,13 +591,19 @@ func (o *Object) stat(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
+	return o.decodeMetadata(ctx, res)
+}
+
+// decodeMetadata updates info fields in the Object according to HTTP response headers
+func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	o.size = rest.ParseSizeFromHeaders(res.Header)
+
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -653,6 +649,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
+	if err = o.decodeMetadata(ctx, res); err != nil {
+		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
+	}
 	return res.Body, nil
 }
 

backend/http/http_internal_test.go

@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -41,12 +41,12 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := ioutil.ReadDir(filesPath)
+	fileList, err := os.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -194,31 +194,66 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	f, tidy := prepare(t)
+	m, tidy := prepareServer(t)
 	defer tidy()
 
-	o, err := f.NewObject(context.Background(), "four/under four.txt")
-	require.NoError(t, err)
+	for _, head := range []bool{false, true} {
+		if !head {
+			m.Set("no_head", "true")
+		}
+		f, err := NewFs(context.Background(), remoteName, "", m)
+		require.NoError(t, err)
 
-	// Test normal read
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	if lineEndSize == 2 {
-		assert.Equal(t, "beetroot\r\n", string(data))
-	} else {
-		assert.Equal(t, "beetroot\n", string(data))
+		for _, rangeRead := range []bool{false, true} {
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			if !head {
+				// Test mod time is still indeterminate
+				tObj := o.ModTime(context.Background())
+				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
+
+				// Test file size is still indeterminate
+				assert.Equal(t, int64(-1), o.Size())
+			}
+
+			var data []byte
+			if !rangeRead {
+				// Test normal read
+				fd, err := o.Open(context.Background())
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				if lineEndSize == 2 {
+					assert.Equal(t, "beetroot\r\n", string(data))
+				} else {
+					assert.Equal(t, "beetroot\n", string(data))
+				}
+			} else {
+				// Test with range request
+				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				assert.Equal(t, "eetro", string(data))
+			}
+
+			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+			require.NoError(t, err)
+			tFile := fi.ModTime()
+
+			// Test the time is always correct on the object after file open
+			tObj := o.ModTime(context.Background())
+			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+			if !rangeRead {
+				// Test the file size
+				assert.Equal(t, int64(len(data)), o.Size())
+			}
+		}
 	}
-
-	// Test with range request
-	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-	require.NoError(t, err)
-	data, err = ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, "eetro", string(data))
 }
 
 func TestMimeType(t *testing.T) {

backend/hubic/auth.go

@@ -1,62 +0,0 @@
-package hubic
-
-import (
-	"context"
-	"net/http"
-	"time"
-
-	"github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/fs"
-)
-
-// auth is an authenticator for swift
-type auth struct {
-	f *Fs
-}
-
-// newAuth creates a swift authenticator
-func newAuth(f *Fs) *auth {
-	return &auth{
-		f: f,
-	}
-}
-
-// Request constructs an http.Request for authentication
-//
-// returns nil for not needed
-func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Request, err error) {
-	const retries = 10
-	for try := 1; try <= retries; try++ {
-		err = a.f.getCredentials(context.TODO())
-		if err == nil {
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-		fs.Debugf(a.f, "retrying auth request %d/%d: %v", try, retries, err)
-	}
-	return nil, err
-}
-
-// Response parses the result of an http request
-func (a *auth) Response(ctx context.Context, resp *http.Response) error {
-	return nil
-}
-
-// The public storage URL - set Internal to true to read
-// internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string { // nolint
-	return a.f.credentials.Endpoint
-}
-
-// The access token
-func (a *auth) Token() string {
-	return a.f.credentials.Token
-}
-
-// The CDN url if available
-func (a *auth) CdnUrl() string { // nolint
-	return ""
-}
-
-// Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)

backend/hubic/hubic.go

@@ -1,200 +0,0 @@
-// Package hubic provides an interface to the Hubic object storage
-// system.
-package hubic
-
-// This uses the normal swift mechanism to update the credentials and
-// ignores the expires field returned by the Hubic API.  This may need
-// to be revisited after some actual experience.
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"time"
-
-	swiftLib "github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/backend/swift"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/oauthutil"
-	"golang.org/x/oauth2"
-)
-
-const (
-	rcloneClientID              = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
-	rcloneEncryptedClientSecret = "leZKCcqy9movLhDWLVXX8cSLp_FzoiAPeEJOIOMRw1A5RuC4iLEPDYPWVF46adC_MVonnLdVEOTHVstfBOZ_lY4WNp8CK_YWlpRZ9diT5YI"
-)
-
-// Globals
-var (
-	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
-		Scopes: []string{
-			"credentials.r", // Read OpenStack credentials
-		},
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://api.hubic.com/oauth/auth/",
-			TokenURL: "https://api.hubic.com/oauth/token/",
-		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
-	}
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "hubic",
-		Description: "Hubic",
-		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
-			})
-		},
-		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
-	})
-}
-
-// credentials is the JSON returned from the Hubic API to read the
-// OpenStack credentials
-type credentials struct {
-	Token    string `json:"token"`    // OpenStack token
-	Endpoint string `json:"endpoint"` // OpenStack endpoint
-	Expires  string `json:"expires"`  // Expires date - e.g. "2015-11-09T14:24:56+01:00"
-}
-
-// Fs represents a remote hubic
-type Fs struct {
-	fs.Fs                    // wrapped Fs
-	features    *fs.Features // optional features
-	client      *http.Client // client for oauth api
-	credentials credentials  // returned from the Hubic API
-	expires     time.Time    // time credentials expire
-}
-
-// Object describes a swift object
-type Object struct {
-	*swift.Object
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.Object.String()
-}
-
-// ------------------------------------------------------------
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	if f.Fs == nil {
-		return "Hubic"
-	}
-	return fmt.Sprintf("Hubic %s", f.Fs.String())
-}
-
-// getCredentials reads the OpenStack Credentials using the Hubic API
-//
-// The credentials are read into the Fs
-func (f *Fs) getCredentials(ctx context.Context) (err error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "https://api.hubic.com/1.0/account/credentials", nil)
-	if err != nil {
-		return err
-	}
-	resp, err := f.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer fs.CheckClose(resp.Body, &err)
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
-		return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
-	}
-	decoder := json.NewDecoder(resp.Body)
-	var result credentials
-	err = decoder.Decode(&result)
-	if err != nil {
-		return err
-	}
-	// fs.Debugf(f, "Got credentials %+v", result)
-	if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
-		return errors.New("couldn't read token, result and expired from credentials")
-	}
-	f.credentials = result
-	expires, err := time.Parse(time.RFC3339, result.Expires)
-	if err != nil {
-		return err
-	}
-	f.expires = expires
-	fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
-	return nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	client, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to configure Hubic: %w", err)
-	}
-
-	f := &Fs{
-		client: client,
-	}
-
-	// Make the swift Connection
-	ci := fs.GetConfig(ctx)
-	c := &swiftLib.Connection{
-		Auth:           newAuth(f),
-		ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * ci.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(ctx),
-	}
-	err = c.Authenticate(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error authenticating swift connection: %w", err)
-	}
-
-	// Parse config into swift.Options struct
-	opt := new(swift.Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Make inner swift Fs from the connection
-	swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true)
-	if err != nil && err != fs.ErrorIsFile {
-		return nil, err
-	}
-	f.Fs = swiftFs
-	f.features = f.Fs.Features().Wrap(f)
-	return f, err
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// UnWrap returns the Fs that this Fs is wrapping
-func (f *Fs) UnWrap() fs.Fs {
-	return f.Fs
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs        = (*Fs)(nil)
-	_ fs.UnWrapper = (*Fs)(nil)
-)

backend/hubic/hubic_test.go

@@ -1,19 +0,0 @@
-// Test Hubic filesystem interface
-package hubic_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/hubic"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:          "TestHubic:",
-		NilObject:           (*hubic.Object)(nil),
-		SkipFsCheckWrap:     true,
-		SkipObjectCheckWrap: true,
-	})
-}

backend/internetarchive/internetarchive_test.go

@@ -0,0 +1,17 @@
+// Test internetarchive filesystem interface
+package internetarchive_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/internetarchive"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestIA:lesmi-rclone-test/",
+		NilObject:  (*internetarchive.Object)(nil),
+	})
+}

backend/jottacloud/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Jottacloud API.
 package api
 
 import (
@@ -8,42 +9,69 @@ import (
 )
 
 const (
-	// default time format for almost all request and responses
-	timeFormat = "2006-01-02-T15:04:05Z0700"
-	// the API server seems to use a different format
-	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
+	// default time format historically used for all request and responses.
+	// Similar to time.RFC3339, but with an extra '-' in front of 'T',
+	// and no ':' separator in timezone offset. Some newer endpoints have
+	// moved to proper time.RFC3339 conformant format instead.
+	jottaTimeFormat = "2006-01-02-T15:04:05Z0700"
 )
 
-// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
-type Time time.Time
-
-// UnmarshalXML turns XML into a Time
-func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+// unmarshalXML turns XML into a Time
+func unmarshalXMLTime(d *xml.Decoder, start xml.StartElement, timeFormat string) (time.Time, error) {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
-		return err
+		return time.Time{}, err
 	}
 	if v == "" {
-		*t = Time(time.Time{})
-		return nil
+		return time.Time{}, nil
 	}
 	newTime, err := time.Parse(timeFormat, v)
 	if err == nil {
-		*t = Time(newTime)
+		return newTime, nil
 	}
+	return time.Time{}, err
+}
+
+// JottaTime represents time values in the classic API using a custom RFC3339 like format
+type JottaTime time.Time
+
+// String returns JottaTime string in Jottacloud classic format
+func (t JottaTime) String() string { return time.Time(t).Format(jottaTimeFormat) }
+
+// UnmarshalXML turns XML into a JottaTime
+func (t *JottaTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	tm, err := unmarshalXMLTime(d, start, jottaTimeFormat)
+	*t = JottaTime(tm)
 	return err
 }
 
-// MarshalXML turns a Time into XML
-func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+// MarshalXML turns a JottaTime into XML
+func (t *JottaTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	return e.EncodeElement(t.String(), start)
 }
 
-// Return Time string in Jottacloud format
-func (t Time) String() string { return time.Time(t).Format(timeFormat) }
+// Rfc3339Time represents time values in the newer APIs using standard RFC3339 format
+type Rfc3339Time time.Time
 
-// APIString returns Time string in Jottacloud API format
-func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
+// String returns Rfc3339Time string in Jottacloud RFC3339 format
+func (t Rfc3339Time) String() string { return time.Time(t).Format(time.RFC3339) }
+
+// UnmarshalXML turns XML into a Rfc3339Time
+func (t *Rfc3339Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	tm, err := unmarshalXMLTime(d, start, time.RFC3339)
+	*t = Rfc3339Time(tm)
+	return err
+}
+
+// MarshalXML turns a Rfc3339Time into XML
+func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	return e.EncodeElement(t.String(), start)
+}
+
+// MarshalJSON turns a Rfc3339Time into JSON
+func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
+}
 
 // LoginToken is struct representing the login token generated in the WebUI
 type LoginToken struct {
@@ -122,16 +150,11 @@ type AllocateFileResponse struct {
 
 // UploadResponse after an upload
 type UploadResponse struct {
-	Name      string      `json:"name"`
-	Path      string      `json:"path"`
-	Kind      string      `json:"kind"`
-	ContentID string      `json:"content_id"`
-	Bytes     int64       `json:"bytes"`
-	Md5       string      `json:"md5"`
-	Created   int64       `json:"created"`
-	Modified  int64       `json:"modified"`
-	Deleted   interface{} `json:"deleted"`
-	Mime      string      `json:"mime"`
+	Path      string `json:"path"`
+	ContentID string `json:"content_id"`
+	Bytes     int64  `json:"bytes"`
+	Md5       string `json:"md5"`
+	Modified  int64  `json:"modified"`
 }
 
 // DeviceRegistrationResponse is the response to registering a device
@@ -338,9 +361,9 @@ type JottaFolder struct {
 	Name       string        `xml:"name,attr"`
 	Deleted    Flag          `xml:"deleted,attr"`
 	Path       string        `xml:"path"`
-	CreatedAt  Time          `xml:"created"`
-	ModifiedAt Time          `xml:"modified"`
-	Updated    Time          `xml:"updated"`
+	CreatedAt  JottaTime     `xml:"created"`
+	ModifiedAt JottaTime     `xml:"modified"`
+	Updated    JottaTime     `xml:"updated"`
 	Folders    []JottaFolder `xml:"folders>folder"`
 	Files      []JottaFile   `xml:"files>file"`
 }
@@ -365,17 +388,17 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
 // JottaFile represents a Jottacloud file
 type JottaFile struct {
 	XMLName         xml.Name
-	Name            string `xml:"name,attr"`
-	Deleted         Flag   `xml:"deleted,attr"`
-	PublicURI       string `xml:"publicURI"`
-	PublicSharePath string `xml:"publicSharePath"`
-	State           string `xml:"currentRevision>state"`
-	CreatedAt       Time   `xml:"currentRevision>created"`
-	ModifiedAt      Time   `xml:"currentRevision>modified"`
-	Updated         Time   `xml:"currentRevision>updated"`
-	Size            int64  `xml:"currentRevision>size"`
-	MimeType        string `xml:"currentRevision>mime"`
-	MD5             string `xml:"currentRevision>md5"`
+	Name            string    `xml:"name,attr"`
+	Deleted         Flag      `xml:"deleted,attr"`
+	PublicURI       string    `xml:"publicURI"`
+	PublicSharePath string    `xml:"publicSharePath"`
+	State           string    `xml:"currentRevision>state"`
+	CreatedAt       JottaTime `xml:"currentRevision>created"`
+	ModifiedAt      JottaTime `xml:"currentRevision>modified"`
+	Updated         JottaTime `xml:"currentRevision>updated"`
+	Size            int64     `xml:"currentRevision>size"`
+	MimeType        string    `xml:"currentRevision>mime"`
+	MD5             string    `xml:"currentRevision>md5"`
 }
 
 // Error is a custom Error for wrapping Jottacloud error responses

backend/jottacloud/jottacloud.go

@@ -1,3 +1,4 @@
+// Package jottacloud provides an interface to the Jottacloud storage system.
 package jottacloud
 
 import (
@@ -11,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -46,9 +46,9 @@ const (
 	decayConstant      = 2 // bigger for slower decay, exponential
 	defaultDevice      = "Jotta"
 	defaultMountpoint  = "Archive"
-	rootURL            = "https://jfs.jottacloud.com/jfs/"
+	jfsURL             = "https://jfs.jottacloud.com/jfs/"
 	apiURL             = "https://api.jottacloud.com/"
-	baseURL            = "https://www.jottacloud.com/"
+	wwwURL             = "https://www.jottacloud.com/"
 	cachePrefix        = "rclone-jcmd5-"
 	configDevice       = "device"
 	configMountpoint   = "mountpoint"
@@ -127,7 +127,7 @@ func init() {
 func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
 	switch config.State {
 	case "":
-		return fs.ConfigChooseFixed("auth_type_done", "config_type", `Authentication type.`, []fs.OptionExample{{
+		return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{
 			Value: "standard",
 			Help:  "Standard authentication.\nUse this if you're a normal Jottacloud user.",
 		}, {
@@ -145,7 +145,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 		return fs.ConfigGoto(config.Result)
 	case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
 		m.Set("configVersion", fmt.Sprint(configVersion))
-		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\n\nGenerate here: https://www.jottacloud.com/web/secure")
+		return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure")
 	case "standard_token":
 		loginToken := config.Result
 		m.Set(configClientID, defaultClientID)
@@ -191,7 +191,7 @@ machines.`)
 		m.Set("auth_code", "")
 		return fs.ConfigGoto("legacy_do_auth")
 	case "legacy_auth_code":
-		authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
+		authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
 		m.Set("auth_code", authCode)
 		return fs.ConfigGoto("legacy_do_auth")
 	case "legacy_do_auth":
@@ -262,7 +262,11 @@ machines.`)
 			},
 		})
 	case "choose_device":
-		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
+		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
+Choosing no, the default, will let you access the storage used for the archive
+section of the official Jottacloud client. If you instead want to access the
+sync or the backup section, for example, you must choose yes.`)
+
 	case "choose_device_query":
 		if config.Result != "true" {
 			m.Set(configDevice, "")
@@ -273,43 +277,139 @@ machines.`)
 		if err != nil {
 			return nil, err
 		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
 		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
 
 		cust, err := getCustomerInfo(ctx, apiSrv)
 		if err != nil {
 			return nil, err
 		}
-		m.Set(configUsername, cust.Username)
 
-		acc, err := getDriveInfo(ctx, srv, cust.Username)
+		acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
 		if err != nil {
 			return nil, err
 		}
-		return fs.ConfigChoose("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) {
-			return acc.Devices[i].Name, ""
+
+		deviceNames := make([]string, len(acc.Devices))
+		for i, dev := range acc.Devices {
+			if i > 0 && dev.Name == defaultDevice {
+				// Insert the special Jotta device as first entry, making it the default choice.
+				copy(deviceNames[1:i+1], deviceNames[0:i])
+				deviceNames[0] = dev.Name
+			} else {
+				deviceNames[i] = dev.Name
+			}
+		}
+
+		help := fmt.Sprintf(`The device to use. In standard setup the built-in %s device is used,
+which contains predefined mountpoints for archive, sync etc. All other devices
+are treated as backup devices by the official Jottacloud client. You may create
+a new by entering a unique name.`, defaultDevice)
+		return fs.ConfigChoose("choose_device_result", "config_device", help, len(deviceNames), func(i int) (string, string) {
+			return deviceNames[i], ""
 		})
 	case "choose_device_result":
 		device := config.Result
-		m.Set(configDevice, device)
 
 		oAuthClient, _, err := getOAuthClient(ctx, name, m)
 		if err != nil {
 			return nil, err
 		}
-		srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
 
-		username, _ := m.Get(configUsername)
-		dev, err := getDeviceInfo(ctx, srv, path.Join(username, device))
+		cust, err := getCustomerInfo(ctx, apiSrv)
 		if err != nil {
 			return nil, err
 		}
-		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) {
+
+		acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
+		if err != nil {
+			return nil, err
+		}
+		isNew := true
+		for _, dev := range acc.Devices {
+			if strings.EqualFold(dev.Name, device) { // If device name exists with different casing we prefer the existing (not sure if and how the api handles the opposite)
+				device = dev.Name // Prefer same casing as existing, e.g. if user entered "jotta" we use the standard casing "Jotta" instead
+				isNew = false
+				break
+			}
+		}
+		var dev *api.JottaDevice
+		if isNew {
+			fs.Debugf(nil, "Creating new device: %s", device)
+			dev, err = createDevice(ctx, jfsSrv, path.Join(cust.Username, device))
+			if err != nil {
+				return nil, err
+			}
+		}
+		m.Set(configDevice, device)
+
+		if !isNew {
+			dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		var help string
+		if device == defaultDevice {
+			// With built-in Jotta device the mountpoint choice is exclusive,
+			// we do not want to risk any problems by creating new mountpoints on it.
+			help = fmt.Sprintf(`The mountpoint to use on the built-in device %s.
+The standard setup is to use the %s mountpoint. Most other mountpoints
+have very limited support in rclone and should generally be avoided.`, defaultDevice, defaultMountpoint)
+			return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
+				return dev.MountPoints[i].Name, ""
+			})
+		}
+		help = fmt.Sprintf(`The mountpoint to use on the non-standard device %s.
+You may create a new by entering a unique name.`, device)
+		return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
 			return dev.MountPoints[i].Name, ""
 		})
 	case "choose_device_mountpoint":
 		mountpoint := config.Result
+
+		oAuthClient, _, err := getOAuthClient(ctx, name, m)
+		if err != nil {
+			return nil, err
+		}
+		jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		cust, err := getCustomerInfo(ctx, apiSrv)
+		if err != nil {
+			return nil, err
+		}
+
+		device, _ := m.Get(configDevice)
+
+		dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
+		if err != nil {
+			return nil, err
+		}
+		isNew := true
+		for _, mnt := range dev.MountPoints {
+			if strings.EqualFold(mnt.Name, mountpoint) {
+				mountpoint = mnt.Name
+				isNew = false
+				break
+			}
+		}
+
+		if isNew {
+			if device == defaultDevice {
+				return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err)
+			}
+			fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint)
+			_, err := createMountPoint(ctx, jfsSrv, path.Join(cust.Username, device, mountpoint))
+			if err != nil {
+				return nil, err
+			}
+		}
 		m.Set(configMountpoint, mountpoint)
+
 		return fs.ConfigGoto("end")
 	case "end":
 		// All the config flows end up here in case we need to carry on with something
@@ -332,16 +432,17 @@ type Options struct {
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name         string
-	root         string
-	user         string
-	opt          Options
-	features     *fs.Features
-	endpointURL  string
-	srv          *rest.Client
-	apiSrv       *rest.Client
-	pacer        *fs.Pacer
-	tokenRenewer *oauthutil.Renew // renew the token on expiry
+	name             string
+	root             string
+	user             string
+	opt              Options
+	features         *fs.Features
+	fileEndpoint     string
+	allocateEndpoint string
+	jfsSrv           *rest.Client
+	apiSrv           *rest.Client
+	pacer            *fs.Pacer
+	tokenRenewer     *oauthutil.Renew // renew the token on expiry
 }
 
 // Object describes a jottacloud object
@@ -519,7 +620,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
 	values.Set("client_id", defaultClientID)
 	values.Set("grant_type", "password")
 	values.Set("password", loginToken.AuthToken)
-	values.Set("scope", "offline_access+openid")
+	values.Set("scope", "openid offline_access")
 	values.Set("username", loginToken.Username)
 	values.Encode()
 	opts = rest.Opts{
@@ -588,15 +689,47 @@ func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *ap
 	return info, nil
 }
 
-// setEndpointURL generates the API endpoint URL
-func (f *Fs) setEndpointURL() {
+// createDevice makes a device
+func createDevice(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       urlPathEscape(path),
+		Parameters: url.Values{},
+	}
+
+	opts.Parameters.Set("type", "WORKSTATION")
+
+	_, err = srv.CallXML(ctx, &opts, nil, &info)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create device: %w", err)
+	}
+	return info, nil
+}
+
+// createMountPoint makes a mount point
+func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info *api.JottaMountPoint, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   urlPathEscape(path),
+	}
+
+	_, err = srv.CallXML(ctx, &opts, nil, &info)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create mountpoint: %w", err)
+	}
+	return info, nil
+}
+
+// setEndpoints generates the API endpoints
+func (f *Fs) setEndpoints() {
 	if f.opt.Device == "" {
 		f.opt.Device = defaultDevice
 	}
 	if f.opt.Mountpoint == "" {
 		f.opt.Mountpoint = defaultMountpoint
 	}
-	f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
+	f.fileEndpoint = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
+	f.allocateEndpoint = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint)
 }
 
 // readMetaDataForPath reads the metadata from the path
@@ -608,7 +741,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
 	var result api.JottaFile
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -649,17 +782,34 @@ func errorHandler(resp *http.Response) error {
 
 // Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
 func urlPathEscape(in string) string {
-	return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
+	return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
 }
 
 // filePathRaw returns an unescaped file path (f.root, file)
-func (f *Fs) filePathRaw(file string) string {
-	return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
+// Optionally made absolute by prefixing with "/", typically required when used
+// as request parameter instead of the path (which is relative to some root url).
+func (f *Fs) filePathRaw(file string, absolute bool) string {
+	prefix := ""
+	if absolute {
+		prefix = "/"
+	}
+	return path.Join(prefix, f.fileEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
 // filePath returns an escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
-	return urlPathEscape(f.filePathRaw(file))
+	return urlPathEscape(f.filePathRaw(file, false))
+}
+
+// allocatePathRaw returns an unescaped allocate file path (f.root, file)
+// Optionally made absolute by prefixing with "/", typically required when used
+// as request parameter instead of the path (which is relative to some root url).
+func (f *Fs) allocatePathRaw(file string, absolute bool) string {
+	prefix := ""
+	if absolute {
+		prefix = "/"
+	}
+	return path.Join(prefix, f.allocateEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
 // Jottacloud requires the grant_type 'refresh_token' string
@@ -671,7 +821,7 @@ func (f *Fs) filePath(file string) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
+		refreshBody, err := io.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -681,7 +831,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -692,12 +842,12 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	if ok {
 		ver, err = strconv.Atoi(version)
 		if err != nil {
-			return nil, nil, errors.New("Failed to parse config version")
+			return nil, nil, errors.New("failed to parse config version")
 		}
 		ok = (ver == configVersion) || (ver == legacyConfigVersion)
 	}
 	if !ok {
-		return nil, nil, errors.New("Outdated config - please reconfigure this backend")
+		return nil, nil, errors.New("outdated config - please reconfigure this backend")
 	}
 
 	baseClient := fshttp.NewClient(ctx)
@@ -743,7 +893,7 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	// Create OAuth Client
 	oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Failed to configure Jottacloud oauth client: %w", err)
+		return nil, nil, fmt.Errorf("failed to configure Jottacloud oauth client: %w", err)
 	}
 	return oAuthClient, ts, nil
 }
@@ -769,7 +919,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:   name,
 		root:   root,
 		opt:    *opt,
-		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
+		jfsSrv: rest.NewClient(oAuthClient).SetRoot(jfsURL),
 		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
 		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
@@ -779,7 +929,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ReadMimeType:            true,
 		WriteMimeType:           false,
 	}).Fill(ctx, f)
-	f.srv.SetErrorHandler(errorHandler)
+	f.jfsSrv.SetErrorHandler(errorHandler)
 	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
 		f.features.ListR = nil
 	}
@@ -798,7 +948,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 	f.user = cust.Username
-	f.setEndpointURL()
+	f.setEndpoints()
 
 	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
@@ -864,7 +1014,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
 	opts.Parameters.Set("mkDir", "true")
 
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &jf)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -893,7 +1043,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var resp *http.Response
 	var result api.JottaFolder
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -932,26 +1082,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
-type listStreamTime time.Time
-
-func (c *listStreamTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	var v string
-	if err := d.DecodeElement(&v, &start); err != nil {
-		return err
-	}
-	t, err := time.Parse(time.RFC3339, v)
-	if err != nil {
-		return err
-	}
-	*c = listStreamTime(t)
-	return nil
-}
-
-func (c listStreamTime) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"%s\"", time.Time(c).Format(time.RFC3339))), nil
-}
-
-func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
+func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
 
 	type stats struct {
 		Folders int `xml:"folders"`
@@ -960,12 +1091,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 	var expected, actual stats
 
 	type xmlFile struct {
-		Path     string         `xml:"path"`
-		Name     string         `xml:"filename"`
-		Checksum string         `xml:"md5"`
-		Size     int64          `xml:"size"`
-		Modified listStreamTime `xml:"modified"`
-		Created  listStreamTime `xml:"created"`
+		Path     string          `xml:"path"`
+		Name     string          `xml:"filename"`
+		Checksum string          `xml:"md5"`
+		Size     int64           `xml:"size"`
+		Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
+		Created  api.Rfc3339Time `xml:"created"`
 	}
 
 	type xmlFolder struct {
@@ -987,8 +1118,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 		})
 	}
 
+	// liststream paths are /mountpoint/root/path
+	// so the returned paths should have /mountpoint/root/ trimmed
+	// as the caller is expecting path.
+	pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
 	trimPathPrefix := func(p string) string {
-		p = strings.TrimPrefix(p, trimPrefix)
+		p = strings.TrimPrefix(p, pathPrefix)
 		p = strings.TrimPrefix(p, "/")
 		return p
 	}
@@ -1045,7 +1180,7 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 
 	if expected.Folders != actual.Folders ||
 		expected.Files != actual.Files {
-		return fmt.Errorf("Invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
+		return fmt.Errorf("invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
 	}
 	return nil
 }
@@ -1066,16 +1201,12 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
+		resp, err = f.jfsSrv.Call(ctx, &opts)
 		if err != nil {
 			return shouldRetry(ctx, resp, err)
 		}
 
-		// liststream paths are /mountpoint/root/path
-		// so the returned paths should have /mountpoint/root/ trimmed
-		// as the caller is expecting path.
-		trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
-		err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
+		err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
 			if d.Remote() == dir {
 				return nil
 			}
@@ -1116,13 +1247,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if f.opt.Device != "Jotta" {
-		return nil, errors.New("upload not supported for devices other than Jotta")
-	}
 	o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size())
 	return o, o.Update(ctx, in, src, options...)
 }
@@ -1132,10 +1260,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	// chop off trailing / if it exists
-	if strings.HasSuffix(dirPath, "/") {
-		dirPath = dirPath[:len(dirPath)-1]
-	}
-	parent := path.Dir(dirPath)
+	parent := path.Dir(strings.TrimSuffix(dirPath, "/"))
 	if parent == "." {
 		parent = ""
 	}
@@ -1183,7 +1308,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
+		resp, err = f.jfsSrv.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1210,6 +1335,45 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.purgeCheck(ctx, dir, false)
 }
 
+// createOrUpdate tries to make remote file match without uploading.
+// If the remote file exists, and has matching size and md5, only
+// timestamps are updated. If the file does not exist or does does
+// not match size and md5, but matching content can be constructed
+// from deduplication, the file will be updated/created. If the file
+// is currently in trash, but can be made to match, it will be
+// restored. Returns ErrorObjectNotFound if upload will be necessary
+// to get a matching remote file.
+func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         f.filePath(file),
+		Parameters:   url.Values{},
+		ExtraHeaders: make(map[string]string),
+	}
+
+	opts.Parameters.Set("cphash", "true")
+
+	fileDate := api.JottaTime(modTime).String()
+	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
+	opts.ExtraHeaders["JMd5"] = md5
+	opts.ExtraHeaders["JCreated"] = fileDate
+	opts.ExtraHeaders["JModified"] = fileDate
+
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
+		return shouldRetry(ctx, resp, err)
+	})
+
+	if apiErr, ok := err.(*api.Error); ok {
+		// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
+		if apiErr.StatusCode == http.StatusNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+	}
+	return info, nil
+}
+
 // copyOrMoves copies or moves directories or files depending on the method parameter
 func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
@@ -1218,11 +1382,11 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, dest))))
+	opts.Parameters.Set(method, f.filePathRaw(dest, true))
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1233,9 +1397,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1253,6 +1417,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
+	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
+	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
+		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
+		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("couldn't copy file: %w", err)
 	}
@@ -1263,9 +1433,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1325,7 +1495,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
+	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.fileEndpoint, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
 	if err != nil {
 		return fmt.Errorf("couldn't move directory: %w", err)
@@ -1350,7 +1520,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	var resp *http.Response
 	var result api.JottaFile
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
 		return shouldRetry(ctx, resp, err)
 	})
 
@@ -1376,19 +1546,19 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return "", errors.New("couldn't create public link - no uri received")
 	}
 	if result.PublicSharePath != "" {
-		webLink := joinPath(baseURL, result.PublicSharePath)
+		webLink := joinPath(wwwURL, result.PublicSharePath)
 		fs.Debugf(nil, "Web link: %s", webLink)
 	} else {
 		fs.Debugf(nil, "No web link received")
 	}
-	directLink := joinPath(baseURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
+	directLink := joinPath(wwwURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
 	fs.Debugf(nil, "Direct link: %s", directLink)
 	return directLink, nil
 }
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
-	info, err := getDriveInfo(ctx, f.srv, f.user)
+	info, err := getDriveInfo(ctx, f.jfsSrv, f.user)
 	if err != nil {
 		return nil, err
 	}
@@ -1554,40 +1724,19 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 		return err
 	}
 
-	// prepare allocate request with existing metadata but changed timestamps
-	var resp *http.Response
-	var options []fs.OpenOption
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "files/v1/allocate",
-		Options:      options,
-		ExtraHeaders: make(map[string]string),
-	}
-	fileDate := api.Time(modTime).APIString()
-	var request = api.AllocateFileRequest{
-		Bytes:    o.size,
-		Created:  fileDate,
-		Modified: fileDate,
-		Md5:      o.md5,
-		Path:     path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
-	}
-
-	// send it
-	var response api.AllocateFileResponse
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(ctx, resp, err)
-	})
+	// request check/update with existing metadata and new modtime
+	// (note that if size/md5 does not match, the file content will
+	// also be modified if deduplication is possible, i.e. it is
+	// important to use correct/latest values)
+	_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
 	if err != nil {
+		if err == fs.ErrorObjectNotFound {
+			// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
+			return errors.New("metadata did not match")
+		}
 		return err
 	}
 
-	// check response
-	if response.State != "COMPLETED" {
-		// could be the file was modified (size/md5 changed) between readMetaData and the allocate request
-		return errors.New("metadata did not match")
-	}
-
 	// update local metadata
 	o.modTime = modTime
 	return nil
@@ -1612,7 +1761,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	opts.Parameters.Set("mode", "bin")
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
+		resp, err = o.fs.jfsSrv.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1639,7 +1788,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = ioutil.TempFile("", cachePrefix)
+		tempFile, err = os.CreateTemp("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1667,7 +1816,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = ioutil.ReadAll(teeReader)
+		inData, err = io.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1680,7 +1829,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
@@ -1725,7 +1874,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options:      options,
 		ExtraHeaders: make(map[string]string),
 	}
-	fileDate := api.Time(src.ModTime(ctx)).APIString()
+	fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
 
 	// the allocate request
 	var request = api.AllocateFileRequest{
@@ -1733,7 +1882,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Created:  fileDate,
 		Modified: fileDate,
 		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
+		Path:     o.fs.allocatePathRaw(o.remote, true),
 	}
 
 	// send it
@@ -1764,7 +1913,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		_, err = io.CopyN(io.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}
@@ -1803,7 +1952,7 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 	}
 
 	return o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
+		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
 		return shouldRetry(ctx, resp, err)
 	})
 }

backend/koofr/koofr.go

@@ -1,3 +1,4 @@
+// Package koofr provides an interface to the Koofr storage system.
 package koofr
 
 import (
@@ -351,9 +352,9 @@ func NewFsFromOptions(ctx context.Context, name, root string, opt *Options) (ff
 	}
 	if f.mountID == "" {
 		if opt.MountID == "" {
-			return nil, errors.New("Failed to find primary mount")
+			return nil, errors.New("failed to find primary mount")
 		}
-		return nil, errors.New("Failed to find mount " + opt.MountID)
+		return nil, errors.New("failed to find mount " + opt.MountID)
 	}
 	rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root))
 	if err == nil && rootFile.Type != "dir" {
@@ -667,7 +668,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	//
 	// https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F
 	//
-	// I am not sure about meaning of "path" parameter; in my expriments
+	// I am not sure about meaning of "path" parameter; in my experiments
 	// it is always "%2F", and omitting it or putting any other value
 	// results in 404.
 	//

backend/local/about_windows.go

@@ -17,8 +17,12 @@ var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSp
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var available, total, free int64
+	root, e := syscall.UTF16PtrFromString(f.root)
+	if e != nil {
+		return nil, fmt.Errorf("failed to read disk usage: %w", e)
+	}
 	_, _, e1 := getFreeDiskSpace.Call(
-		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))),
+		uintptr(unsafe.Pointer(root)),
 		uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user
 		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
 		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes

backend/local/local.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -22,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -42,9 +42,22 @@ func init() {
 		Description: "Local Disk",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
+		MetadataInfo: &fs.MetadataInfo{
+			System: systemMetadataInfo,
+			Help: `Depending on which OS is in use the local backend may return only some
+of the system metadata. Setting system metadata is supported on all
+OSes but setting user metadata is only supported on linux, freebsd,
+netbsd, macOS and Solaris. It is **not** supported on Windows yet
+([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)).
+
+User metadata is stored as extended attributes (which may not be
+supported by all file systems) under the "user.*" prefix.
+`,
+		},
 		Options: []fs.Option{{
 			Name:     "nounc",
 			Help:     "Disable UNC (long path names) conversion on Windows.",
+			Default:  false,
 			Advanced: runtime.GOOS != "windows",
 			Examples: []fs.OptionExample{{
 				Value: "true",
@@ -110,8 +123,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy -
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
@@ -221,15 +234,16 @@ type Options struct {
 
 // Fs represents a local filesystem rooted at root
 type Fs struct {
-	name        string              // the name of the remote
-	root        string              // The root directory (OS path)
-	opt         Options             // parsed config options
-	features    *fs.Features        // optional features
-	dev         uint64              // device number of root node
-	precisionOk sync.Once           // Whether we need to read the precision
-	precision   time.Duration       // precision of local filesystem
-	warnedMu    sync.Mutex          // used for locking access to 'warned'.
-	warned      map[string]struct{} // whether we have warned about this string
+	name           string              // the name of the remote
+	root           string              // The root directory (OS path)
+	opt            Options             // parsed config options
+	features       *fs.Features        // optional features
+	dev            uint64              // device number of root node
+	precisionOk    sync.Once           // Whether we need to read the precision
+	precision      time.Duration       // precision of local filesystem
+	warnedMu       sync.Mutex          // used for locking access to 'warned'.
+	warned         map[string]struct{} // whether we have warned about this string
+	xattrSupported int32               // whether xattrs are supported (atomic access)
 
 	// do os.Lstat or os.Stat
 	lstat        func(name string) (os.FileInfo, error)
@@ -273,12 +287,19 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		dev:    devUnset,
 		lstat:  os.Lstat,
 	}
+	if xattrSupported {
+		f.xattrSupported = 1
+	}
 	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
 		IsLocal:                 true,
 		SlowHash:                true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:             true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -423,6 +444,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx)
+
 	fsDirPath := f.localPath(dir)
 	_, err = os.Stat(fsDirPath)
 	if err != nil {
@@ -473,6 +496,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 						continue
 					}
 					if fierr != nil {
+						// Don't report errors on any file names that are excluded
+						if useFilter {
+							newRemote := f.cleanRemote(dir, name)
+							if !filter.IncludeRemote(newRemote) {
+								continue
+							}
+						}
 						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
@@ -506,6 +536,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				}
 				mode = fi.Mode()
 			}
+			// Don't include non directory if not included
+			// we leave directory filtering to the layer above
+			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
+				continue
+			}
 			if fi.IsDir() {
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
@@ -610,7 +645,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := ioutil.TempFile("", "rclone")
+	fd, err := os.CreateTemp("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -679,9 +714,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -903,7 +938,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", fmt.Errorf("hash: failed to open: %w", err)
 		}
 		var hashes map[hash.Type]string
-		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
+		hashes, err = hash.StreamTypes(readers.NewContextReader(ctx, in), hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
 			return "", fmt.Errorf("hash: failed to read: %w", err)
@@ -937,17 +972,22 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 	return o.modTime
 }
 
+// Set the atime and ltime of the object
+func (o *Object) setTimes(atime, mtime time.Time) (err error) {
+	if o.translatedLink {
+		err = lChtimes(o.path, atime, mtime)
+	} else {
+		err = os.Chtimes(o.path, atime, mtime)
+	}
+	return err
+}
+
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if o.fs.opt.NoSetModTime {
 		return nil
 	}
-	var err error
-	if o.translatedLink {
-		err = lChtimes(o.path, modTime, modTime)
-	} else {
-		err = os.Chtimes(o.path, modTime, modTime)
-	}
+	err := o.setTimes(modTime, modTime)
 	if err != nil {
 		return err
 	}
@@ -1032,7 +1072,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read
@@ -1222,6 +1262,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
+	// Fetch and set metadata if --metadata is in use
+	meta, err := fs.GetMetadataOptions(ctx, src, options)
+	if err != nil {
+		return fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	err = o.writeMetadata(meta)
+	if err != nil {
+		return fmt.Errorf("failed to set metadata: %w", err)
+	}
+
 	// ReRead info now that we have finished
 	return o.lstat()
 }
@@ -1320,31 +1370,56 @@ func (o *Object) Remove(ctx context.Context) error {
 	return remove(o.path)
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	metadata, err = o.getXattr()
+	if err != nil {
+		return nil, err
+	}
+	err = o.readMetadataFromFile(&metadata)
+	if err != nil {
+		return nil, err
+	}
+	return metadata, nil
+}
+
+// Write the metadata on the object
+func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
+	err = o.setXattr(metadata)
+	if err != nil {
+		return err
+	}
+	err = o.writeMetadataToFile(metadata)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS == "windows" {
-		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
+	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
+		if !filepath.IsAbs(s) {
 			s2, err := filepath.Abs(s)
 			if err == nil {
 				s = s2
 			}
+		} else {
+			s = filepath.Clean(s)
 		}
+	}
+	if runtime.GOOS == "windows" {
 		s = filepath.ToSlash(s)
 		vol := filepath.VolumeName(s)
 		s = vol + enc.FromStandardPath(s[len(vol):])
 		s = filepath.FromSlash(s)
-
 		if !noUNC {
 			// Convert to UNC
 			s = file.UNCPath(s)
 		}
 		return s
 	}
-	if !filepath.IsAbs(s) {
-		s2, err := filepath.Abs(s)
-		if err == nil {
-			s = s2
-		}
-	}
 	s = enc.FromStandardPath(s)
 	return s
 }
@@ -1359,4 +1434,5 @@ var (
 	_ fs.Commander      = &Fs{}
 	_ fs.OpenWriterAter = &Fs{}
 	_ fs.Object         = &Object{}
+	_ fs.Metadataer     = &Object{}
 )

backend/local/local_internal_test.go

@@ -3,15 +3,19 @@ package local
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
+	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
+	"sort"
 	"testing"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
@@ -146,7 +150,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
+	contents, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -154,7 +158,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
+	contents, err = io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
@@ -188,7 +192,7 @@ func TestHashOnUpdate(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
 
-	// Reupload it with diferent contents but same size and timestamp
+	// Reupload it with different contents but same size and timestamp
 	var b = bytes.NewBufferString("CONTENT")
 	src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
 	err = o.Update(ctx, b, src)
@@ -229,3 +233,171 @@ func TestHashOnDelete(t *testing.T) {
 	_, err = o.Hash(ctx, hash.MD5)
 	require.Error(t, err)
 }
+
+func TestMetadata(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	const filePath = "metafile.txt"
+	when := time.Now()
+	const dayLength = len("2001-01-01")
+	whenRFC := when.Format(time.RFC3339Nano)
+	r.WriteFile(filePath, "metadata file contents", when)
+	f := r.Flocal.(*Fs)
+
+	// Get the object
+	obj, err := f.NewObject(ctx, filePath)
+	require.NoError(t, err)
+	o := obj.(*Object)
+
+	features := f.Features()
+
+	var hasXID, hasAtime, hasBtime bool
+	switch runtime.GOOS {
+	case "darwin", "freebsd", "netbsd", "linux":
+		hasXID, hasAtime, hasBtime = true, true, true
+	case "openbsd", "solaris":
+		hasXID, hasAtime = true, true
+	case "windows":
+		hasAtime, hasBtime = true, true
+	case "plan9", "js":
+		// nada
+	default:
+		t.Errorf("No test cases for OS %q", runtime.GOOS)
+	}
+
+	assert.True(t, features.ReadMetadata)
+	assert.True(t, features.WriteMetadata)
+	assert.Equal(t, xattrSupported, features.UserMetadata)
+
+	t.Run("Xattr", func(t *testing.T) {
+		if !xattrSupported {
+			t.Skip()
+		}
+		m, err := o.getXattr()
+		require.NoError(t, err)
+		assert.Nil(t, m)
+
+		inM := fs.Metadata{
+			"potato":  "chips",
+			"cabbage": "soup",
+		}
+		err = o.setXattr(inM)
+		require.NoError(t, err)
+
+		m, err = o.getXattr()
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+		assert.Equal(t, inM, m)
+	})
+
+	checkTime := func(m fs.Metadata, key string, when time.Time) {
+		mt, ok := o.parseMetadataTime(m, key)
+		assert.True(t, ok)
+		dt := mt.Sub(when)
+		precision := time.Second
+		assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v", key, dt, precision))
+	}
+
+	checkInt := func(m fs.Metadata, key string, base int) int {
+		value, ok := o.parseMetadataInt(m, key, base)
+		assert.True(t, ok)
+		return value
+	}
+	t.Run("Read", func(t *testing.T) {
+		m, err := o.Metadata(ctx)
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+
+		// All OSes have these
+		checkInt(m, "mode", 8)
+		checkTime(m, "mtime", when)
+
+		assert.Equal(t, len(whenRFC), len(m["mtime"]))
+		assert.Equal(t, whenRFC[:dayLength], m["mtime"][:dayLength])
+
+		if hasAtime {
+			checkTime(m, "atime", when)
+		}
+		if hasBtime {
+			checkTime(m, "btime", when)
+		}
+		if hasXID {
+			checkInt(m, "uid", 10)
+			checkInt(m, "gid", 10)
+		}
+	})
+
+	t.Run("Write", func(t *testing.T) {
+		newAtimeString := "2011-12-13T14:15:16.999999999Z"
+		newAtime := fstest.Time(newAtimeString)
+		newMtimeString := "2011-12-12T14:15:16.999999999Z"
+		newMtime := fstest.Time(newMtimeString)
+		newBtimeString := "2011-12-11T14:15:16.999999999Z"
+		newBtime := fstest.Time(newBtimeString)
+		newM := fs.Metadata{
+			"mtime": newMtimeString,
+			"atime": newAtimeString,
+			"btime": newBtimeString,
+			// Can't test uid, gid without being root
+			"mode":   "0767",
+			"potato": "wedges",
+		}
+		err := o.writeMetadata(newM)
+		require.NoError(t, err)
+
+		m, err := o.Metadata(ctx)
+		require.NoError(t, err)
+		assert.NotNil(t, m)
+
+		mode := checkInt(m, "mode", 8)
+		if runtime.GOOS != "windows" {
+			assert.Equal(t, 0767, mode&0777, fmt.Sprintf("mode wrong - expecting 0767 got 0%o", mode&0777))
+		}
+
+		checkTime(m, "mtime", newMtime)
+		if hasAtime {
+			checkTime(m, "atime", newAtime)
+		}
+		if haveSetBTime {
+			checkTime(m, "btime", newBtime)
+		}
+		if xattrSupported {
+			assert.Equal(t, "wedges", m["potato"])
+		}
+	})
+
+}
+
+func TestFilter(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	r.WriteFile("included", "included file", when)
+	r.WriteFile("excluded", "excluded file", when)
+	f := r.Flocal.(*Fs)
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included"))
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[excluded included]", fmt.Sprint(entries))
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[included]", fmt.Sprint(entries))
+}

backend/local/local_test.go

@@ -11,7 +11,8 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "",
-		NilObject:  (*local.Object)(nil),
+		RemoteName:  "",
+		NilObject:   (*local.Object)(nil),
+		QuickTestOK: true,
 	})
 }

backend/local/metadata.go

@@ -0,0 +1,138 @@
+package local
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+const metadataTimeFormat = time.RFC3339Nano
+
+// system metadata keys which this backend owns
+//
+// not all values supported on all OSes
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+	"mode": {
+		Help:    "File type and mode",
+		Type:    "octal, unix style",
+		Example: "0100664",
+	},
+	"uid": {
+		Help:    "User ID of owner",
+		Type:    "decimal number",
+		Example: "500",
+	},
+	"gid": {
+		Help:    "Group ID of owner",
+		Type:    "decimal number",
+		Example: "500",
+	},
+	"rdev": {
+		Help:    "Device ID (if special file)",
+		Type:    "hexadecimal",
+		Example: "1abc",
+	},
+	"atime": {
+		Help:    "Time of last access",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+	"mtime": {
+		Help:    "Time of last modification",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+	"btime": {
+		Help:    "Time of file birth (creation)",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999999999Z07:00",
+	},
+}
+
+// parse a time string from metadata with key
+func (o *Object) parseMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
+	value, ok := m[key]
+	if ok {
+		var err error
+		t, err = time.Parse(metadataTimeFormat, value)
+		if err != nil {
+			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
+			ok = false
+		}
+	}
+	return t, ok
+}
+
+// parse am int from metadata with key and base
+func (o *Object) parseMetadataInt(m fs.Metadata, key string, base int) (result int, ok bool) {
+	value, ok := m[key]
+	if ok {
+		var err error
+		result64, err := strconv.ParseInt(value, base, 64)
+		if err != nil {
+			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
+			ok = false
+		}
+		result = int(result64)
+	}
+	return result, ok
+}
+
+// Write the metadata into the file
+//
+// It isn't possible to set the ctime and btime under Unix
+func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
+	var err error
+	atime, atimeOK := o.parseMetadataTime(m, "atime")
+	mtime, mtimeOK := o.parseMetadataTime(m, "mtime")
+	btime, btimeOK := o.parseMetadataTime(m, "btime")
+	if atimeOK || mtimeOK {
+		if atimeOK && !mtimeOK {
+			mtime = atime
+		}
+		if !atimeOK && mtimeOK {
+			atime = mtime
+		}
+		err = o.setTimes(atime, mtime)
+		if err != nil {
+			outErr = fmt.Errorf("failed to set times: %w", err)
+		}
+	}
+	if haveSetBTime {
+		if btimeOK {
+			err = setBTime(o.path, btime)
+			if err != nil {
+				outErr = fmt.Errorf("failed to set birth (creation) time: %w", err)
+			}
+		}
+	}
+	uid, hasUID := o.parseMetadataInt(m, "uid", 10)
+	gid, hasGID := o.parseMetadataInt(m, "gid", 10)
+	if hasUID {
+		// FIXME should read UID and GID of current user and only attempt to set it if different
+		if !hasGID {
+			gid = uid
+		}
+		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
+			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
+		} else {
+			err = os.Chown(o.path, uid, gid)
+			if err != nil {
+				outErr = fmt.Errorf("failed to change ownership: %w", err)
+			}
+		}
+	}
+	mode, hasMode := o.parseMetadataInt(m, "mode", 8)
+	if hasMode {
+		err = os.Chmod(o.path, os.FileMode(mode))
+		if err != nil {
+			outErr = fmt.Errorf("failed to change permissions: %w", err)
+		}
+	}
+	// FIXME not parsing rdev yet
+	return outErr
+}

backend/local/metadata_bsd.go

@@ -0,0 +1,38 @@
+//go:build darwin || freebsd || netbsd
+// +build darwin freebsd netbsd
+
+package local
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	info, err := o.fs.lstat(o.path)
+	if err != nil {
+		return err
+	}
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		fs.Debugf(o, "didn't return Stat_t as expected")
+		return nil
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
+	}
+	setTime := func(key string, t syscall.Timespec) {
+		m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atimespec)
+	setTime("mtime", stat.Mtimespec)
+	setTime("btime", stat.Birthtimespec)
+	return nil
+}

backend/local/metadata_linux.go

@@ -0,0 +1,102 @@
+//go:build linux
+// +build linux
+
+package local
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"golang.org/x/sys/unix"
+)
+
+var (
+	statxCheckOnce         sync.Once
+	readMetadataFromFileFn func(o *Object, m *fs.Metadata) (err error)
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	statxCheckOnce.Do(func() {
+		// Check statx() is available as it was only introduced in kernel 4.11
+		// If not, fall back to fstatat() which was introduced in 2.6.16 which is guaranteed for all Go versions
+		var stat unix.Statx_t
+		if unix.Statx(unix.AT_FDCWD, ".", 0, unix.STATX_ALL, &stat) != unix.ENOSYS {
+			readMetadataFromFileFn = readMetadataFromFileStatx
+		} else {
+			readMetadataFromFileFn = readMetadataFromFileFstatat
+		}
+	})
+	return readMetadataFromFileFn(o, m)
+}
+
+// Read the metadata from the file into metadata where possible
+func readMetadataFromFileStatx(o *Object, m *fs.Metadata) (err error) {
+	flags := unix.AT_SYMLINK_NOFOLLOW
+	if o.fs.opt.FollowSymlinks {
+		flags = 0
+	}
+	var stat unix.Statx_t
+	//  statx() was added to Linux in kernel 4.11
+	err = unix.Statx(unix.AT_FDCWD, o.path, flags, (0 |
+		unix.STATX_TYPE | // Want stx_mode & S_IFMT
+		unix.STATX_MODE | // Want stx_mode & ~S_IFMT
+		unix.STATX_UID | // Want stx_uid
+		unix.STATX_GID | // Want stx_gid
+		unix.STATX_ATIME | // Want stx_atime
+		unix.STATX_MTIME | // Want stx_mtime
+		unix.STATX_CTIME | // Want stx_ctime
+		unix.STATX_BTIME), // Want stx_btime
+		&stat)
+	if err != nil {
+		return err
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev_major != 0 || stat.Rdev_minor != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", uint64(stat.Rdev_major)<<32|uint64(stat.Rdev_minor)))
+	}
+	setTime := func(key string, t unix.StatxTimestamp) {
+		m.Set(key, time.Unix(t.Sec, int64(t.Nsec)).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atime)
+	setTime("mtime", stat.Mtime)
+	setTime("btime", stat.Btime)
+	return nil
+}
+
+// Read the metadata from the file into metadata where possible
+func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
+	flags := unix.AT_SYMLINK_NOFOLLOW
+	if o.fs.opt.FollowSymlinks {
+		flags = 0
+	}
+	var stat unix.Stat_t
+	// fstatat() was added to Linux in  kernel  2.6.16
+	// Go only supports 2.6.32 or later
+	err = unix.Fstatat(unix.AT_FDCWD, o.path, &stat, flags)
+	if err != nil {
+		return err
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
+	}
+	setTime := func(key string, t unix.Timespec) {
+		// The types of t.Sec and t.Nsec vary from int32 to int64 on
+		// different Linux architectures so we need to cast them to
+		// int64 here and hence need to quiet the linter about
+		// unecessary casts.
+		//
+		// nolint: unconvert
+		m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atim)
+	setTime("mtime", stat.Mtim)
+	return nil
+}

backend/local/metadata_other.go

@@ -0,0 +1,21 @@
+//go:build plan9 || js
+// +build plan9 js
+
+package local
+
+import (
+	"fmt"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	info, err := o.fs.lstat(o.path)
+	if err != nil {
+		return err
+	}
+	m.Set("mode", fmt.Sprintf("%0o", info.Mode()))
+	m.Set("mtime", info.ModTime().Format(metadataTimeFormat))
+	return nil
+}

backend/local/metadata_unix.go

@@ -0,0 +1,37 @@
+//go:build openbsd || solaris
+// +build openbsd solaris
+
+package local
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	info, err := o.fs.lstat(o.path)
+	if err != nil {
+		return err
+	}
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		fs.Debugf(o, "didn't return Stat_t as expected")
+		return nil
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
+	}
+	setTime := func(key string, t syscall.Timespec) {
+		m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atim)
+	setTime("mtime", stat.Mtim)
+	return nil
+}

backend/local/metadata_windows.go

@@ -0,0 +1,34 @@
+//go:build windows
+// +build windows
+
+package local
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Read the metadata from the file into metadata where possible
+func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	info, err := o.fs.lstat(o.path)
+	if err != nil {
+		return err
+	}
+	stat, ok := info.Sys().(*syscall.Win32FileAttributeData)
+	if !ok {
+		fs.Debugf(o, "didn't return Win32FileAttributeData as expected")
+		return nil
+	}
+	// FIXME do something with stat.FileAttributes ?
+	m.Set("mode", fmt.Sprintf("%0o", info.Mode()))
+	setTime := func(key string, t syscall.Filetime) {
+		m.Set(key, time.Unix(0, t.Nanoseconds()).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.LastAccessTime)
+	setTime("mtime", stat.LastWriteTime)
+	setTime("btime", stat.CreationTime)
+	return nil
+}

backend/local/remove_test.go

@@ -1,7 +1,6 @@
 package local
 
 import (
-	"io/ioutil"
 	"os"
 	"sync"
 	"testing"
@@ -13,7 +12,7 @@ import (
 
 // Check we can remove an open file
 func TestRemove(t *testing.T) {
-	fd, err := ioutil.TempFile("", "rclone-remove-test")
+	fd, err := os.CreateTemp("", "rclone-remove-test")
 	require.NoError(t, err)
 	name := fd.Name()
 	defer func() {

backend/local/setbtime.go

@@ -0,0 +1,16 @@
+//go:build !windows
+// +build !windows
+
+package local
+
+import (
+	"time"
+)
+
+const haveSetBTime = false
+
+// setBTime changes the birth time of the file passed in
+func setBTime(name string, btime time.Time) error {
+	// Does nothing
+	return nil
+}

backend/local/setbtime_windows.go

@@ -0,0 +1,28 @@
+//go:build windows
+// +build windows
+
+package local
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+const haveSetBTime = true
+
+// setBTime sets the birth time of the file passed in
+func setBTime(name string, btime time.Time) (err error) {
+	h, err := syscall.Open(name, os.O_RDWR, 0755)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		closeErr := syscall.Close(h)
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	bFileTime := syscall.NsecToFiletime(btime.UnixNano())
+	return syscall.SetFileTime(h, &bFileTime, nil, nil)
+}

backend/local/xattr.go

@@ -0,0 +1,116 @@
+//go:build !openbsd && !plan9
+// +build !openbsd,!plan9
+
+package local
+
+import (
+	"fmt"
+	"strings"
+	"sync/atomic"
+	"syscall"
+
+	"github.com/pkg/xattr"
+	"github.com/rclone/rclone/fs"
+)
+
+const (
+	xattrPrefix    = "user." // FIXME is this correct for all unixes?
+	xattrSupported = xattr.XATTR_SUPPORTED
+)
+
+// Check to see if the error supplied is a not supported error, and if
+// so, disable xattrs
+func (f *Fs) xattrIsNotSupported(err error) bool {
+	xattrErr, ok := err.(*xattr.Error)
+	if !ok {
+		return false
+	}
+	// Xattrs not supported can be ENOTSUP or ENOATTR or EINVAL (on Solaris)
+	if xattrErr.Err == syscall.EINVAL || xattrErr.Err == syscall.ENOTSUP || xattrErr.Err == xattr.ENOATTR {
+		// Show xattrs not supported
+		if atomic.CompareAndSwapInt32(&f.xattrSupported, 1, 0) {
+			fs.Errorf(f, "xattrs not supported - disabling: %v", err)
+		}
+		return true
+	}
+	return false
+}
+
+// getXattr returns the extended attributes for an object
+//
+// It doesn't return any attributes owned by this backend in
+// metadataKeys
+func (o *Object) getXattr() (metadata fs.Metadata, err error) {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
+		return nil, nil
+	}
+	var list []string
+	if o.fs.opt.FollowSymlinks {
+		list, err = xattr.List(o.path)
+	} else {
+		list, err = xattr.LList(o.path)
+	}
+	if err != nil {
+		if o.fs.xattrIsNotSupported(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to read xattr: %w", err)
+	}
+	if len(list) == 0 {
+		return nil, nil
+	}
+	metadata = make(fs.Metadata, len(list))
+	for _, k := range list {
+		var v []byte
+		if o.fs.opt.FollowSymlinks {
+			v, err = xattr.Get(o.path, k)
+		} else {
+			v, err = xattr.LGet(o.path, k)
+		}
+		if err != nil {
+			if o.fs.xattrIsNotSupported(err) {
+				return nil, nil
+			}
+			return nil, fmt.Errorf("failed to read xattr key %q: %w", k, err)
+		}
+		k = strings.ToLower(k)
+		if !strings.HasPrefix(k, xattrPrefix) {
+			continue
+		}
+		k = k[len(xattrPrefix):]
+		if _, found := systemMetadataInfo[k]; found {
+			continue
+		}
+		metadata[k] = string(v)
+	}
+	return metadata, nil
+}
+
+// setXattr sets the metadata on the file Xattrs
+//
+// It doesn't set any attributes owned by this backend in metadataKeys
+func (o *Object) setXattr(metadata fs.Metadata) (err error) {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
+		return nil
+	}
+	for k, value := range metadata {
+		k = strings.ToLower(k)
+		if _, found := systemMetadataInfo[k]; found {
+			continue
+		}
+		k = xattrPrefix + k
+		v := []byte(value)
+		if o.fs.opt.FollowSymlinks {
+			err = xattr.Set(o.path, k, v)
+		} else {
+			err = xattr.LSet(o.path, k, v)
+		}
+		if err != nil {
+			if o.fs.xattrIsNotSupported(err) {
+				return nil
+			}
+			return fmt.Errorf("failed to set xattr key %q: %w", k, err)
+		}
+	}
+	return nil
+}

backend/local/xattr_unsupported.go

@@ -0,0 +1,21 @@
+//go:build openbsd || plan9
+// +build openbsd plan9
+
+// The pkg/xattr module doesn't compile for openbsd or plan9
+package local
+
+import "github.com/rclone/rclone/fs"
+
+const (
+	xattrSupported = false
+)
+
+// getXattr returns the extended attributes for an object
+func (o *Object) getXattr() (metadata fs.Metadata, err error) {
+	return nil, nil
+}
+
+// setXattr sets the metadata on the file Xattrs
+func (o *Object) setXattr(metadata fs.Metadata) (err error) {
+	return nil
+}