vfs: fix reload: failed to add virtual dir entry: file does not exist

This error happened on a restart of the VFS with files to upload into
a new directory on a bucket based backend. Rclone was assuming that
directories created before the restart would still exist, but this is
a bad assumption for bucket based backends which don't really have
directories.

This change creates the pretend directory and thus the directory cache
if the parent directory does not exist when adding a virtual on a
backend which can't have empty directories.

See: https://forum.rclone.org/t/that-pesky-failed-to-reload-error-message/34527

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -8,29 +8,31 @@ name: build
 on:
   push:
     branches:
-      - '*'
+      - '**'
     tags:
-      - '*'
+      - '**'
   pull_request:
   workflow_dispatch:
     inputs:
       manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
         required: true
         default: true
 
 jobs:
   build:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.20'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -41,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.20'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.19.x'
+            go: '1.20'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -57,14 +59,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.19.x'
+            go: '1.20'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.19.x'
+            go: '1.20'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -74,20 +76,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.20'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.17
+          - job_name: go1.18
             os: ubuntu-latest
-            go: '1.17.x'
+            go: '1.18'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.18
+          - job_name: go1.19
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.19'
             quicktest: true
             racequicktest: true
 
@@ -102,9 +104,8 @@ jobs:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 
@@ -123,7 +124,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse libfuse-dev rpm pkg-config
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
@@ -216,10 +217,10 @@ jobs:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
         # Deploy binaries if enabled in config && not a PR && not a fork
-        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "lint"
     runs-on: ubuntu-latest
@@ -234,8 +235,21 @@ jobs:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
+      # Run govulncheck on the latest go version, the one we build binaries with
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+          check-latest: true
+
+      - name: Install govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+
+      - name: Scan for vulnerabilities
+        run: govulncheck ./...
+
   android:
-    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "android-all"
     runs-on: ubuntu-latest
@@ -248,9 +262,9 @@ jobs:
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: '1.20'
 
       - name: Go module cache
         uses: actions/cache@v3
@@ -338,4 +352,4 @@ jobs:
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork
-        if: github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,61 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v2
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v4
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v2
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Build and publish image
+              uses: docker/build-push-action@v4
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha
+                cache-to: type=gha,mode=max
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252

.github/workflows/build_publish_docker_image.yml

@@ -1,26 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v3
-              with:
-                fetch-depth: 0
-            - name: Build and publish image
-              uses: ilteoood/docker_buildx@1.1.0
-              with:
-                tag: beta
-                imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                publish: true
-                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
-                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/winget.yml

@@ -0,0 +1,14 @@
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: windows-latest # Action can only run on Windows
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.golangci.yml

@@ -2,15 +2,17 @@
 
 linters:
   enable:
-    - deadcode
     - errcheck
     - goimports
     - revive
     - ineffassign
-    - structcheck
-    - varcheck
     - govet
     - unconvert
+    - staticcheck
+    - gosimple
+    - stylecheck
+    - unused
+    - misspell
     #- prealloc
     #- maligned
   disable-all: true
@@ -25,6 +27,30 @@ issues:
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
+  exclude-rules:
+
+    - linters:
+      - staticcheck
+      text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
+
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 10m
+
+linters-settings:
+  revive:
+    rules:
+      - name: unreachable-code
+        disabled: true
+      - name: unused-parameter
+        disabled: true
+      - name: empty-block
+        disabled: true
+      - name: redefines-builtin-id
+        disabled: true
+      - name: superfluous-else
+        disabled: true
+  stylecheck:
+    # Only enable the checks performed by the staticcheck stand-alone tool,
+    # as documented here: https://staticcheck.io/docs/configuration/options/#checks
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]

Dockerfile

@@ -11,7 +11,7 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse tzdata && \
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

MAINTAINERS.md

@@ -16,6 +16,7 @@ Current active maintainers of rclone are:
 | Max Sum          | @Max-Sum          | union backend                |
 | Fred             | @creativeprojects | seafile backend              |
 | Caleb Case       | @calebcase        | storj backend                |
+| wiserain         | @wiserain         | pikpak backend               |
 
 **This is a work in progress Draft**
 

Makefile

@@ -81,6 +81,9 @@ quicktest:
 racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
+compiletest:
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
+
 # Do source code quality checks
 check:	rclone
 	@echo "-- START CODE QUALITY REPORT -------------------------------"

README.md

@@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
+  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
@@ -50,6 +51,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
@@ -65,6 +67,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
+  * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)

RELEASE.md

@@ -10,7 +10,7 @@ This file describes how to make the various kinds of releases
 ## Making a release
 
   * git checkout master # see below for stable branch
-  * git pull
+  * git pull # IMPORTANT
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
@@ -21,6 +21,7 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
+  * git push origin # without --follow-tags so it doesn't push the tag if it fails
   * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
@@ -74,8 +75,7 @@ Set vars
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * git branch ${BASE_TAG} ${BASE_TAG}-stable
-  * git co ${BASE_TAG}-stable
+  * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
   * make startstable
 
 Now

VERSION

@@ -1 +1 @@
-v1.60.0
+v1.63.0

backend/all/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
+	_ "github.com/rclone/rclone/backend/pikpak"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"

backend/azureblob/azureblob_internal_test.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
+//go:build !plan9 && !solaris && !js && go1.18
+// +build !plan9,!solaris,!js,go1.18
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,12 +1,11 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
+//go:build !plan9 && !solaris && !js && go1.18
+// +build !plan9,!solaris,!js,go1.18
 
 package azureblob
 
 import (
-	"context"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -17,10 +16,12 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:    "TestAzureBlob:",
-		NilObject:     (*Object)(nil),
-		TiersToTest:   []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{},
+		RemoteName:  "TestAzureBlob:",
+		NilObject:   (*Object)(nil),
+		TiersToTest: []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: defaultChunkSize,
+		},
 	})
 }
 
@@ -32,36 +33,6 @@ var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 )
 
-// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
-func TestServicePrincipalFileSuccess(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "password": "my secret",
-    "tenant": "my active directory tenant ID"
-}
-`
-	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	if assert.NoError(t, err) {
-		assert.NotNil(t, tokenRefresher)
-	}
-}
-
-// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
-func TestServicePrincipalFileFailure(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "tenant": "my active directory tenant ID"
-}
-`
-	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	assert.Error(t, err)
-	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
-}
-
 func TestValidateAccessTier(t *testing.T) {
 	tests := map[string]struct {
 		accessTier string

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js
-// +build plan9 solaris js
+//go:build plan9 || solaris || js || !go1.18
+// +build plan9 solaris js !go1.18
 
 package azureblob

backend/azureblob/imds.go

@@ -1,137 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-)
-
-const (
-	azureResource      = "https://storage.azure.com"
-	imdsAPIVersion     = "2018-02-01"
-	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
-)
-
-// This custom type is used to add the port the test server has bound to
-// to the request context.
-type testPortKey string
-
-type msiIdentifierType int
-
-const (
-	msiClientID msiIdentifierType = iota
-	msiObjectID
-	msiResourceID
-)
-
-type userMSI struct {
-	Type  msiIdentifierType
-	Value string
-}
-
-type httpError struct {
-	Response *http.Response
-}
-
-func (e httpError) Error() string {
-	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
-}
-
-// GetMSIToken attempts to obtain an MSI token from the Azure Instance
-// Metadata Service.
-func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
-	// Attempt to get an MSI token; silently continue if unsuccessful.
-	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
-	result := adal.Token{}
-	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
-	if err != nil {
-		fs.Debugf(nil, "Failed to create request: %v", err)
-		return result, err
-	}
-	params := req.URL.Query()
-	params.Set("resource", azureResource)
-	params.Set("api-version", imdsAPIVersion)
-
-	// Specify user-assigned identity if requested.
-	if identity != nil {
-		switch identity.Type {
-		case msiClientID:
-			params.Set("client_id", identity.Value)
-		case msiObjectID:
-			params.Set("object_id", identity.Value)
-		case msiResourceID:
-			params.Set("mi_res_id", identity.Value)
-		default:
-			// If this happens, the calling function and this one don't agree on
-			// what valid ID types exist.
-			return result, fmt.Errorf("unknown MSI identity type specified")
-		}
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// The Metadata header is required by all calls to IMDS.
-	req.Header.Set("Metadata", "true")
-
-	// If this function is run in a test, query the test server instead of IMDS.
-	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
-	if isTest {
-		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
-		req.Host = req.URL.Host
-	}
-
-	// Send request
-	httpClient := fshttp.NewClient(ctx)
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
-	}
-	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
-		if err != nil {
-			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
-		}
-		err = resp.Body.Close()
-		if err != nil {
-			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
-		}
-	}()
-	// Check if the status code indicates success
-	// The request returns 200 currently, add 201 and 202 as well for possible extension.
-	switch resp.StatusCode {
-	case 200, 201, 202:
-		break
-	default:
-		body, _ := ioutil.ReadAll(resp.Body)
-		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
-		return result, httpError{Response: resp}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
-	}
-	// Remove BOM, if any. azcopy does this so I'm following along.
-	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
-
-	// This would be a good place to persist the token if a large number of rclone
-	// invocations are being made in a short amount of time. If the token is
-	// persisted, the azureblob code will need to check for expiry before every
-	// storage API call.
-	err = json.Unmarshal(b, &result)
-	if err != nil {
-		return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err)
-	}
-
-	return result, nil
-}

backend/azureblob/imds_test.go

@@ -1,118 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		err := r.ParseForm()
-		require.NoError(t, err)
-		parameters := r.URL.Query()
-		(*actual)["path"] = r.URL.Path
-		(*actual)["Metadata"] = r.Header.Get("Metadata")
-		(*actual)["method"] = r.Method
-		for paramName := range parameters {
-			(*actual)[paramName] = parameters.Get(paramName)
-		}
-		// Make response.
-		response := adal.Token{}
-		responseBytes, err := json.Marshal(response)
-		require.NoError(t, err)
-		_, err = w.Write(responseBytes)
-		require.NoError(t, err)
-	}
-}
-
-func TestManagedIdentity(t *testing.T) {
-	// test user-assigned identity specifiers to use
-	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
-	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
-	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
-	tests := []struct {
-		identity              *userMSI
-		identityParameterName string
-		expectedAbsent        []string
-	}{
-		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
-		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
-		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
-		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
-	}
-	alwaysExpected := map[string]string{
-		"path":        "/metadata/identity/oauth2/token",
-		"resource":    "https://storage.azure.com",
-		"Metadata":    "true",
-		"api-version": "2018-02-01",
-		"method":      "GET",
-	}
-	for _, test := range tests {
-		actual := make(map[string]string, 10)
-		testServer := httptest.NewServer(handler(t, &actual))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, test.identity)
-		require.NoError(t, err)
-
-		// Validate expected query parameters present
-		expected := make(map[string]string)
-		for k, v := range alwaysExpected {
-			expected[k] = v
-		}
-		if test.identity != nil {
-			expected[test.identityParameterName] = test.identity.Value
-		}
-
-		for key := range expected {
-			value, exists := actual[key]
-			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
-				test.identityParameterName, key) {
-				assert.Equalf(t, expected[key], value,
-					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
-			}
-		}
-
-		// Validate unexpected query parameters absent
-		for _, key := range test.expectedAbsent {
-			_, exists := actual[key]
-			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
-		}
-	}
-}
-
-func errorHandler(resultCode int) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Test error generated", resultCode)
-	}
-}
-
-func TestIMDSErrors(t *testing.T) {
-	errorCodes := []int{404, 429, 500}
-	for _, code := range errorCodes {
-		testServer := httptest.NewServer(errorHandler(code))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, nil)
-		require.Error(t, err)
-		httpErr, ok := err.(httpError)
-		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
-		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
-	}
-}

backend/b2/b2.go

@@ -1221,7 +1221,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 					fs.Errorf(object.Name, "Can't create object %v", err)
 					continue
 				}
-				tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
+				tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
 				err = f.deleteByID(ctx, object.ID, object.Name)
 				checkErr(err)
 				tr.Done(ctx, err)
@@ -1235,7 +1235,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			if err != nil {
 				fs.Errorf(object, "Can't create object %+v", err)
 			}
-			tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
+			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 			if oldOnly && last != remote {
 				// Check current version of the file
 				if object.Action == "hide" {

backend/b2/upload.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
@@ -21,6 +22,7 @@ import (
 	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/sync/errgroup"
 )
@@ -428,18 +430,47 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
 	var (
-		g, gCtx   = errgroup.WithContext(ctx)
-		remaining = up.size
+		g, gCtx    = errgroup.WithContext(ctx)
+		remaining  = up.size
+		uploadPool *pool.Pool
+		ci         = fs.GetConfig(ctx)
 	)
+	// If using large chunk size then make a temporary pool
+	if up.chunkSize <= int64(up.f.opt.ChunkSize) {
+		uploadPool = up.f.pool
+	} else {
+		uploadPool = pool.New(
+			time.Duration(up.f.opt.MemoryPoolFlushTime),
+			int(up.chunkSize),
+			ci.Transfers,
+			up.f.opt.MemoryPoolUseMmap,
+		)
+		defer uploadPool.Flush()
+	}
+	// Get an upload token and a buffer
+	getBuf := func() (buf []byte) {
+		up.f.getBuf(true)
+		if !up.doCopy {
+			buf = uploadPool.Get()
+		}
+		return buf
+	}
+	// Put an upload token and a buffer
+	putBuf := func(buf []byte) {
+		if !up.doCopy {
+			uploadPool.Put(buf)
+		}
+		up.f.putBuf(nil, true)
+	}
 	g.Go(func() error {
 		for part := int64(1); part <= up.parts; part++ {
 			// Get a block of memory from the pool and token which limits concurrency.
-			buf := up.f.getBuf(up.doCopy)
+			buf := getBuf()
 
 			// Fail fast, in case an errgroup managed function returns an error
 			// gCtx is cancelled. There is no point in uploading all the other parts.
 			if gCtx.Err() != nil {
-				up.f.putBuf(buf, up.doCopy)
+				putBuf(buf)
 				return nil
 			}
 
@@ -453,14 +484,14 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 				buf = buf[:reqSize]
 				_, err = io.ReadFull(up.in, buf)
 				if err != nil {
-					up.f.putBuf(buf, up.doCopy)
+					putBuf(buf)
 					return err
 				}
 			}
 
 			part := part // for the closure
 			g.Go(func() (err error) {
-				defer up.f.putBuf(buf, up.doCopy)
+				defer putBuf(buf)
 				if !up.doCopy {
 					err = up.transferChunk(gCtx, part, buf)
 				} else {

backend/box/box.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -27,6 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/golang-jwt/jwt/v4"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -45,7 +46,6 @@ import (
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/jws"
 )
 
 const (
@@ -76,6 +76,11 @@ var (
 	}
 )
 
+type boxCustomClaims struct {
+	jwt.RegisteredClaims
+	BoxSubType string `json:"box_sub_type,omitempty"`
+}
+
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -178,12 +183,12 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
 	client := fshttp.NewClient(ctx)
-	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+	err = jwtutil.Config("box", name, tokenURL, *claims, signingHeaders, queryParams, privateKey, m, client)
 	return err
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := ioutil.ReadFile(configFile)
+	file, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}
@@ -194,34 +199,29 @@ func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
 	return boxConfig, nil
 }
 
-func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) {
+func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomClaims, err error) {
 	val, err := jwtutil.RandomHex(20)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to generate random string for jti: %w", err)
 	}
 
-	claims = &jws.ClaimSet{
-		Iss: boxConfig.BoxAppSettings.ClientID,
-		Sub: boxConfig.EnterpriseID,
-		Aud: tokenURL,
-		Exp: time.Now().Add(time.Second * 45).Unix(),
-		PrivateClaims: map[string]interface{}{
-			"box_sub_type": boxSubType,
-			"aud":          tokenURL,
-			"jti":          val,
+	claims = &boxCustomClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			ID:        val,
+			Issuer:    boxConfig.BoxAppSettings.ClientID,
+			Subject:   boxConfig.EnterpriseID,
+			Audience:  jwt.ClaimStrings{tokenURL},
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
 		},
+		BoxSubType: boxSubType,
 	}
-
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header {
-	signingHeaders := &jws.Header{
-		Algorithm: "RS256",
-		Typ:       "JWT",
-		KeyID:     boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
+func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
+	signingHeaders := map[string]interface{}{
+		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
-
 	return signingHeaders
 }
 

backend/cache/cache.go

@@ -1038,7 +1038,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		}
 		fs.Debugf(dir, "list: remove entry: %v", entryRemote)
 	}
-	entries = nil
+	entries = nil //nolint:ineffassign
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
 	var batchDirectories []*Directory
@@ -1787,7 +1787,7 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
 	}
 }
 
-// StopBackgroundRunners will signall all the runners to stop their work
+// StopBackgroundRunners will signal all the runners to stop their work
 // can be triggered from a terminate signal or from testing between runs
 func (f *Fs) StopBackgroundRunners() {
 	f.cleanupChan <- false

backend/cache/cache_internal_test.go

@@ -11,7 +11,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -102,14 +101,12 @@ func TestMain(m *testing.M) {
 
 func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	id := fmt.Sprintf("tilrair%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	// Instantiate inner fs
 	innerFolder := "inner"
 	runInstance.mkdir(t, rootFs, innerFolder)
-	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs2, boltDb2)
+	rootFs2, _ := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil)
 
 	runInstance.writeObjectString(t, rootFs2, "one", "content")
 	listRoot, err := runInstance.list(t, rootFs, "")
@@ -167,7 +164,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -226,8 +223,7 @@ func TestInternalVfsCache(t *testing.T) {
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -259,8 +255,7 @@ func TestInternalObjWrapFsFound(t *testing.T) {
 
 func TestInternalObjNotFound(t *testing.T) {
 	id := fmt.Sprintf("tionf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
@@ -270,8 +265,7 @@ func TestInternalObjNotFound(t *testing.T) {
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -298,8 +292,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	// write the object
 	runInstance.writeRemoteString(t, rootFs, "one", "one content")
@@ -317,8 +310,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	var err error
 
 	// create some rand test data
@@ -347,8 +339,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -378,8 +369,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -405,8 +395,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -460,8 +449,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 
 func TestInternalMoveWithNotify(t *testing.T) {
 	id := fmt.Sprintf("timwn%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -547,8 +535,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
 
 func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	id := fmt.Sprintf("tincep%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -634,8 +621,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -667,8 +653,7 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"writes": "true"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -689,8 +674,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("timcsr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"workers": "1"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -725,8 +709,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
@@ -763,9 +746,7 @@ func TestInternalBug2117(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
-		map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
 
 	if runInstance.rootIsCrypt {
 		t.Skipf("skipping crypt")
@@ -841,7 +822,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
+		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -866,7 +847,7 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 	return enc
 }
 
-func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) {
+func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
 	for _, s := range config.FileSections() {
@@ -959,10 +940,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		runInstance.cleanupFs(t, f)
+	})
+
 	return f, boltDb
 }
 
-func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
+func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
 	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
@@ -984,7 +970,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := ioutil.TempFile("", "rclonecache-tempfile")
+	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {
@@ -1112,27 +1098,6 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
-func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = in.Close()
-	}()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = out.Close()
-	}()
-
-	_, err = io.Copy(out, in)
-	return err
-}
-
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 

backend/cache/cache_upload_test.go

@@ -21,10 +21,8 @@ import (
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
-		nil,
+	runInstance.newCacheFs(t, remoteName, id, false, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -63,9 +61,7 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -73,19 +69,15 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -119,10 +111,8 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -162,10 +152,8 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
@@ -213,9 +201,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -343,9 +329,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 

backend/cache/plex.go

@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = ioutil.ReadAll(resp.Body)
+							data, err = io.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}

backend/cache/storage_persistent.go

@@ -9,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -473,7 +472,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := ioutil.ReadFile(fp)
+	data, err := os.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -486,7 +485,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := ioutil.WriteFile(filePath, data, os.ModePerm)
+	err := os.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}

backend/chunker/chunker.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -1038,7 +1037,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := ioutil.ReadAll(reader)
+	metadata, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1097,7 +1096,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadAll(reader)
+	data, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err

backend/chunker/chunker_internal_test.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
+		chunkContents, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
+		_, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
+			data, err := io.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -758,8 +758,8 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
+	in := io.NopCloser(bytes.NewBufferString("abc"))
+	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
+	data, err := io.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()

backend/combine/combine.go

@@ -1,4 +1,4 @@
-// Package combine implents a backend to combine multiple remotes in a directory tree
+// Package combine implements a backend to combine multiple remotes in a directory tree
 package combine
 
 /*
@@ -351,7 +351,7 @@ func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream
 	return g.Wait()
 }
 
-// join the elements together but unline path.Join return empty string
+// join the elements together but unlike path.Join return empty string
 func join(elem ...string) string {
 	result := path.Join(elem...)
 	if result == "." {
@@ -631,7 +631,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 	if err != nil {
 		return nil, err
 	}
-	uSrc := operations.NewOverrideRemote(src, uRemote)
+	uSrc := fs.NewOverrideRemote(src, uRemote)
 	var o fs.Object
 	if stream {
 		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)

backend/compress/compress.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -468,7 +467,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := ioutil.TempFile("", "rclone-press-")
+	tempFile, err := os.CreateTemp("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -546,8 +545,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)

backend/crypt/cipher.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
@@ -178,6 +179,7 @@ type Cipher struct {
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
+	passBadBlocks  bool // if set passed bad blocks as zeroed blocks
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -189,7 +191,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 		dirNameEncrypt: dirNameEncrypt,
 	}
 	c.buffers.New = func() interface{} {
-		return make([]byte, blockSize)
+		return new([blockSize]byte)
 	}
 	err := c.Key(password, salt)
 	if err != nil {
@@ -198,11 +200,16 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
+// Call to set bad block pass through
+func (c *Cipher) setPassBadBlocks(passBadBlocks bool) {
+	c.passBadBlocks = passBadBlocks
+}
+
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
 // If salt is "" we use a fixed salt just to make attackers lives
-// slighty harder than using no salt.
+// slightly harder than using no salt.
 //
 // Note that empty password makes all 0x00 keys which is used in the
 // tests.
@@ -230,15 +237,12 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() []byte {
-	return c.buffers.Get().([]byte)
+func (c *Cipher) getBlock() *[blockSize]byte {
+	return c.buffers.Get().(*[blockSize]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf []byte) {
-	if len(buf) != blockSize {
-		panic("bad blocksize returned to pool")
-	}
+func (c *Cipher) putBlock(buf *[blockSize]byte) {
 	c.buffers.Put(buf)
 }
 
@@ -609,7 +613,7 @@ func (n *nonce) pointer() *[fileNonceSize]byte {
 // fromReader fills the nonce from an io.Reader - normally the OSes
 // crypto random number generator
 func (n *nonce) fromReader(in io.Reader) error {
-	read, err := io.ReadFull(in, (*n)[:])
+	read, err := readers.ReadFill(in, (*n)[:])
 	if read != fileNonceSize {
 		return fmt.Errorf("short read of nonce: %w", err)
 	}
@@ -664,8 +668,8 @@ type encrypter struct {
 	in       io.Reader
 	c        *Cipher
 	nonce    nonce
-	buf      []byte
-	readBuf  []byte
+	buf      *[blockSize]byte
+	readBuf  *[blockSize]byte
 	bufIndex int
 	bufSize  int
 	err      error
@@ -690,9 +694,9 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 		}
 	}
 	// Copy magic into buffer
-	copy(fh.buf, fileMagicBytes)
+	copy((*fh.buf)[:], fileMagicBytes)
 	// Copy nonce into buffer
-	copy(fh.buf[fileMagicSize:], fh.nonce[:])
+	copy((*fh.buf)[fileMagicSize:], fh.nonce[:])
 	return fh, nil
 }
 
@@ -707,22 +711,20 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 	if fh.bufIndex >= fh.bufSize {
 		// Read data
 		// FIXME should overlap the reads with a go-routine and 2 buffers?
-		readBuf := fh.readBuf[:blockDataSize]
-		n, err = io.ReadFull(fh.in, readBuf)
+		readBuf := (*fh.readBuf)[:blockDataSize]
+		n, err = readers.ReadFill(fh.in, readBuf)
 		if n == 0 {
-			// err can't be nil since:
-			// n == len(buf) if and only if err == nil.
 			return fh.finish(err)
 		}
 		// possibly err != nil here, but we will process the
-		// data and the next call to ReadFull will return 0, err
+		// data and the next call to ReadFill will return 0, err
 		// Encrypt the block using the nonce
-		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		secretbox.Seal((*fh.buf)[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufSize])
 	fh.bufIndex += n
 	return n, nil
 }
@@ -763,8 +765,8 @@ type decrypter struct {
 	nonce        nonce
 	initialNonce nonce
 	c            *Cipher
-	buf          []byte
-	readBuf      []byte
+	buf          *[blockSize]byte
+	readBuf      *[blockSize]byte
 	bufIndex     int
 	bufSize      int
 	err          error
@@ -782,9 +784,9 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 		limit:   -1,
 	}
 	// Read file header (magic + nonce)
-	readBuf := fh.readBuf[:fileHeaderSize]
-	_, err := io.ReadFull(fh.rc, readBuf)
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
+	readBuf := (*fh.readBuf)[:fileHeaderSize]
+	n, err := readers.ReadFill(fh.rc, readBuf)
+	if n < fileHeaderSize && err == io.EOF {
 		// This read from 0..fileHeaderSize-1 bytes
 		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
 	} else if err != nil {
@@ -845,10 +847,8 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 func (fh *decrypter) fillBuffer() (err error) {
 	// FIXME should overlap the reads with a go-routine and 2 buffers?
 	readBuf := fh.readBuf
-	n, err := io.ReadFull(fh.rc, readBuf)
+	n, err := readers.ReadFill(fh.rc, (*readBuf)[:])
 	if n == 0 {
-		// err can't be nil since:
-		// n == len(buf) if and only if err == nil.
 		return err
 	}
 	// possibly err != nil here, but we will process the data and
@@ -856,18 +856,25 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	// Check header + 1 byte exists
 	if n <= blockHeaderSize {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	_, ok := secretbox.Open((*fh.buf)[:0], (*readBuf)[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
-		return ErrorEncryptedBadBlock
+		if !fh.c.passBadBlocks {
+			return ErrorEncryptedBadBlock
+		}
+		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
+		// Zero out the bad block and continue
+		for i := range (*fh.buf)[:n] {
+			(*fh.buf)[i] = 0
+		}
 	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize
@@ -893,7 +900,7 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	if fh.limit >= 0 && fh.limit < int64(toCopy) {
 		toCopy = int(fh.limit)
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufIndex+toCopy])
 	fh.bufIndex += n
 	if fh.limit >= 0 {
 		fh.limit -= int64(n)
@@ -904,9 +911,8 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-// calculateUnderlying converts an (offset, limit) in a crypted file
-// into an (underlyingOffset, underlyingLimit) for the underlying
-// file.
+// calculateUnderlying converts an (offset, limit) in an encrypted file
+// into an (underlyingOffset, underlyingLimit) for the underlying file.
 //
 // It also returns number of bytes to discard after reading the first
 // block and number of blocks this is from the start so the nonce can

backend/crypt/cipher_test.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -28,14 +27,14 @@ func TestNewNameEncryptionMode(t *testing.T) {
 		{"off", NameEncryptionOff, ""},
 		{"standard", NameEncryptionStandard, ""},
 		{"obfuscate", NameEncryptionObfuscated, ""},
-		{"potato", NameEncryptionOff, "Unknown file name encryption mode \"potato\""},
+		{"potato", NameEncryptionOff, "unknown file name encryption mode \"potato\""},
 	} {
 		actual, actualErr := NewNameEncryptionMode(test.in)
 		assert.Equal(t, actual, test.expected)
 		if test.expectedErr == "" {
 			assert.NoError(t, actualErr)
 		} else {
-			assert.Error(t, actualErr, test.expectedErr)
+			assert.EqualError(t, actualErr, test.expectedErr)
 		}
 	}
 }
@@ -727,7 +726,7 @@ func TestNonceFromReader(t *testing.T) {
 	assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
 	buf = bytes.NewBufferString("123456789abcdefghijklmn")
 	err = x.fromReader(buf)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 func TestNonceFromBuf(t *testing.T) {
@@ -1051,7 +1050,7 @@ func TestRandomSource(t *testing.T) {
 	_, _ = source.Read(buf)
 	sink = newRandomSource(1e8)
 	_, err = io.Copy(sink, source)
-	assert.Error(t, err, "Error in stream")
+	assert.EqualError(t, err, "Error in stream at 1")
 }
 
 type zeroes struct{}
@@ -1073,7 +1072,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1144,15 +1143,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := ioutil.ReadAll(encrypted)
+		out, err := io.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
+		decrypted, err := c.DecryptData(io.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = ioutil.ReadAll(decrypted)
+		out, err = io.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1168,13 +1167,13 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err := c.newEncrypter(z, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
-	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
+	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, (*fh.buf)[:32])
 
 	// Test error path
 	c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
@@ -1187,7 +1186,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1225,7 +1224,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
 		fh, err = c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedFileTooShort.Error())
+		assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
 		assert.Equal(t, 1, cd.closed)
 	}
 
@@ -1233,7 +1232,7 @@ func TestNewDecrypter(t *testing.T) {
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "potato")
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 1, cd.closed)
 
 	// bad magic
@@ -1244,7 +1243,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0copy))
 		fh, err := c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+		assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 		file0copy[i] ^= 0x1
 		assert.Equal(t, 1, cd.closed)
 	}
@@ -1257,12 +1256,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := ioutil.NopCloser(io.MultiReader(in1, in2))
+	in := io.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1274,14 +1273,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
+	plaintext, err := io.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := ioutil.ReadAll(encrypted)
+	ciphertext, err := io.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1300,7 +1299,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1490,14 +1489,16 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = ioutil.ReadAll(fh)
+		_, err = io.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
 			// This would normally produce an error *except* on the first block
 			expectedErr = nil
+		case i <= fileHeaderSize+blockHeaderSize:
+			expectedErr = ErrorEncryptedFileBadHeader
 		default:
-			expectedErr = io.ErrUnexpectedEOF
+			expectedErr = ErrorEncryptedBadBlock
 		}
 		if expectedErr != nil {
 			assert.EqualError(t, err, expectedErr.Error(), what)
@@ -1514,8 +1515,8 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = ioutil.ReadAll(fh)
-	assert.Error(t, err, "potato")
+	_, err = io.ReadAll(fh)
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
 	// Test corrupting the input
@@ -1524,17 +1525,28 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
-			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+			assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = ioutil.ReadAll(fh)
-			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
+			_, err = io.ReadAll(fh)
+			assert.EqualError(t, err, ErrorEncryptedBadBlock.Error())
 		}
 		file16copy[i] ^= 0xFF
 	}
+
+	// Test that we can corrupt a byte and read zeroes if
+	// passBadBlocks is set
+	copy(file16copy, file16)
+	file16copy[len(file16copy)-1] ^= 0xFF
+	c.passBadBlocks = true
+	fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
+	assert.NoError(t, err)
+	buf, err := io.ReadAll(fh)
+	assert.NoError(t, err)
+	assert.Equal(t, make([]byte, 16), buf)
 }
 
 func TestDecrypterClose(t *testing.T) {
@@ -1555,7 +1567,7 @@ func TestDecrypterClose(t *testing.T) {
 
 	// double close
 	err = fh.Close()
-	assert.Error(t, err, ErrorFileClosed.Error())
+	assert.EqualError(t, err, ErrorFileClosed.Error())
 	assert.Equal(t, 1, cd.closed)
 
 	// try again reading the file this time
@@ -1565,7 +1577,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := ioutil.ReadAll(fh)
+	out, err := io.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)
@@ -1582,8 +1594,6 @@ func TestPutGetBlock(t *testing.T) {
 	block := c.getBlock()
 	c.putBlock(block)
 	c.putBlock(block)
-
-	assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) })
 }
 
 func TestKey(t *testing.T) {

backend/crypt/crypt.go

@@ -119,6 +119,15 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
+		}, {
+			Name: "pass_bad_blocks",
+			Help: `If set this will pass bad blocks through as all 0.
+
+This should not be set in normal operation, it should only be set if
+trying to recover a crypted file with errors and it is desired to
+recover as much of the file as possible.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -138,7 +147,7 @@ length and if it's case sensitive.`,
 				},
 				{
 					Value: "base32768",
-					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox)",
 				},
 			},
 			Advanced: true,
@@ -174,6 +183,7 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
+	cipher.setPassBadBlocks(opt.PassBadBlocks)
 	return cipher, nil
 }
 
@@ -235,7 +245,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         cipher.NameEncryptionMode() == NameEncryptionOff,
+		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
 		DuplicateFiles:          true,
 		ReadMimeType:            false, // MimeTypes not supported with crypt
 		WriteMimeType:           false,
@@ -262,6 +272,7 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
+	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
 }
 
@@ -396,6 +407,8 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	ci := fs.GetConfig(ctx)
+
 	if f.opt.NoDataEncryption {
 		o, err := put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
 		if err == nil && o != nil {
@@ -413,6 +426,9 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	// Find a hash the destination supports to compute a hash of
 	// the encrypted data
 	ht := f.Fs.Hashes().GetOne()
+	if ci.IgnoreChecksum {
+		ht = hash.None
+	}
 	var hasher *hash.MultiHasher
 	if ht != hash.None {
 		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
@@ -449,7 +465,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v crypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -1047,10 +1063,11 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	// Get the underlying object if there is one
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
-		// Otherwise likely is an operations.OverrideRemote
+	} else if do, ok := o.ObjectInfo.(*fs.OverrideRemote); ok {
+		// Unwrap if it is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
+		// Otherwise don't unwrap any further
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash

backend/crypt/crypt_internal_test.go

@@ -17,41 +17,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type testWrapper struct {
-	fs.ObjectInfo
-}
-
-// UnWrap returns the Object that this Object is wrapping or nil if it
-// isn't wrapping anything
-func (o testWrapper) UnWrap() fs.Object {
-	if o, ok := o.ObjectInfo.(fs.Object); ok {
-		return o
-	}
-	return nil
-}
-
 // Create a temporary local fs to upload things from
 
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
 	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	}
-	return localFs, cleanup
+	})
+	return localFs
 }
 
 // Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
 	inBuf := bytes.NewBufferString(contents)
 	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
 	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
 	obj, err := f.Put(context.Background(), inBuf, upSrc)
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, obj.Remove(context.Background()))
-	}
-	return obj, cleanup
+	})
+	return obj
 }
 
 // Test the ObjectInfo
@@ -65,11 +52,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 		path = "_wrap"
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
-	obj, cleanupObj := uploadFile(t, localFs, path, contents)
-	defer cleanupObj()
+	obj := uploadFile(t, localFs, path, contents)
 
 	// encrypt the data
 	inBuf := bytes.NewBufferString(contents)
@@ -83,7 +68,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	var oi fs.ObjectInfo = obj
 	if wrap {
 		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = testWrapper{oi}
+		oi = fs.NewOverrideRemote(oi, "new_remote")
 	}
 
 	// wrap the object in a crypt for upload using the nonce we
@@ -116,16 +101,13 @@ func testComputeHash(t *testing.T, f *Fs) {
 		t.Skipf("%v: does not support hashes", f.Fs)
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
 	// Upload a file to localFs as a test object
-	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
-	defer cleanupLocalObj()
+	localObj := uploadFile(t, localFs, path, contents)
 
 	// Upload the same data to the remote Fs also
-	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
-	defer cleanupRemoteObj()
+	remoteObj := uploadFile(t, f, path, contents)
 
 	// Calculate the expected Hash of the remote object
 	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)

backend/drive/drive.go

@@ -14,11 +14,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"path"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -203,7 +202,7 @@ func init() {
 					m.Set("root_folder_id", "appDataFolder")
 				}
 
-				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
+				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
 					return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
 						OAuth2Config: driveConfig,
 					})
@@ -452,7 +451,11 @@ If downloading a file returns the error "This file has been identified
 as malware or spam and cannot be downloaded" with the error code
 "cannotDownloadAbusiveFile" then supply this flag to rclone to
 indicate you acknowledge the risks of downloading the file and rclone
-will download it anyway.`,
+will download it anyway.
+
+Note that if you are using service account it will need Manager
+permission (not Content Manager) to for this flag to work. If the SA
+does not have the right permission, Google will just ignore the flag.`,
 			Advanced: true,
 		}, {
 			Name:     "keep_revision_forever",
@@ -595,6 +598,18 @@ resource key is no needed.
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// Don't encode / as it's a valid name character in drive.
 			Default: encoder.EncodeInvalidUtf8,
+		}, {
+			Name:     "env_auth",
+			Help:     "Get IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 
@@ -651,6 +666,7 @@ type Options struct {
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
 	ResourceKey               string               `config:"resource_key"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote drive server
@@ -758,7 +774,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
-			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
+			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
 				fs.Errorf(f, "Received upload limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
@@ -1108,7 +1124,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -1119,6 +1135,12 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 		if err != nil {
 			return nil, fmt.Errorf("failed to create oauth client from service account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		scopes := driveScopes(opt.Scope)
+		oAuthClient, err = google.DefaultClient(ctx, scopes...)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create client from environment: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
 		if err != nil {
@@ -2877,6 +2899,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			if f.rootFolderID == "appDataFolder" {
 				changesCall.Spaces("appDataFolder")
 			}
+			changesCall.RestrictToMyDrive(!f.opt.SharedWithMe)
 			changeList, err = changesCall.Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
@@ -3323,9 +3346,9 @@ This takes an optional directory to trash which make this easier to
 use via the API.
 
     rclone backend untrash drive:directory
-    rclone backend -i untrash drive:directory subdir
+    rclone backend --interactive untrash drive:directory subdir
 
-Use the -i flag to see what would be restored before restoring it.
+Use the --interactive/-i or --dry-run flag to see what would be restored before restoring it.
 
 Result:
 
@@ -3355,7 +3378,7 @@ component will be used as the file name.
 If the destination is a drive backend then server-side copying will be
 attempted if possible.
 
-Use the -i flag to see what would be copied before copying.
+Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
 `,
 }, {
 	Name:  "exportformats",
@@ -3431,13 +3454,12 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
-		re := regexp.MustCompile(`[^\w_. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
 			upstreams := []string{}
 			names := make(map[string]struct{}, len(drives))
 			for i, drive := range drives {
-				name := re.ReplaceAllString(drive.Name, "_")
+				name := fspath.MakeConfigName(drive.Name)
 				for {
 					if _, found := names[name]; !found {
 						break
@@ -3800,7 +3822,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(data)), nil
+	return io.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,

backend/drive/drive_internal_test.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -78,7 +77,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -244,6 +243,15 @@ func (f *Fs) InternalTestShouldRetry(t *testing.T) {
 	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
 	assert.False(t, quotaExceededRetry)
 	assert.Equal(t, quotaExceededError, expectedQuotaError)
+
+	sqEItem := googleapi.ErrorItem{
+		Reason: "storageQuotaExceeded",
+	}
+	generic403.Errors[0] = sqEItem
+	expectedStorageQuotaError := fserrors.FatalError(&generic403)
+	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, storageQuotaExceededRetry)
+	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
 }
 
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {

backend/dropbox/batcher.go

@@ -13,7 +13,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -140,49 +139,6 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
 	return complete, nil
 }
 
-// finishBatchJobStatus waits for the batch to complete returning completed entries
-func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *files.UploadSessionFinishBatchLaunch) (complete *files.UploadSessionFinishBatchResult, err error) {
-	if launchBatchStatus.AsyncJobId == "" {
-		return nil, errors.New("wait for batch completion: empty job ID")
-	}
-	var batchStatus *files.UploadSessionFinishBatchJobStatus
-	sleepTime := 100 * time.Millisecond
-	const maxSleepTime = 1 * time.Second
-	startTime := time.Now()
-	try := 1
-	for {
-		remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
-		if remaining < 0 {
-			break
-		}
-		err = b.f.pacer.Call(func() (bool, error) {
-			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
-				AsyncJobId: launchBatchStatus.AsyncJobId,
-			})
-			return shouldRetry(ctx, err)
-		})
-		if err != nil {
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
-		} else {
-			if batchStatus.Tag == "complete" {
-				fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
-				return batchStatus.Complete, nil
-			}
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
-		}
-		time.Sleep(sleepTime)
-		sleepTime *= 2
-		if sleepTime > maxSleepTime {
-			sleepTime = maxSleepTime
-		}
-		try++
-	}
-	if err == nil {
-		err = errors.New("batch didn't complete")
-	}
-	return nil, fmt.Errorf("wait for batch failed after %d tries in %v: %w", try, time.Since(startTime), err)
-}
-
 // commit a batch
 func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []chan<- batcherResponse) (err error) {
 	// If commit fails then signal clients if sync

backend/dropbox/dropbox.go

@@ -536,7 +536,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			default:
 				return nil, err
 			}
-			// if the moint failed we have to abort here
+			// if the mount failed we have to abort here
 		}
 		// if the mount succeeded it's now a normal folder in the users root namespace
 		// we disable shared folder mode and proceed normally

backend/fichier/api.go

@@ -473,7 +473,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("didnt got an upload node: %w", err)
+		return nil, fmt.Errorf("didn't get an upload node: %w", err)
 	}
 
 	// fs.Debugf(f, "Got Upload node")

backend/fichier/fichier.go

@@ -333,7 +333,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
 	if size > int64(300e9) {
-		return nil, errors.New("File too big, cant upload")
+		return nil, errors.New("File too big, can't upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles
 	}

backend/filefabric/filefabric.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -1186,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/ftp"
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -70,7 +70,7 @@ func init() {
 When using implicit FTP over TLS the client connects using TLS
 right from the start which breaks compatibility with
 non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTP.`,
+than port 21. Cannot be used in combination with explicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "explicit_tls",
@@ -78,7 +78,7 @@ than port 21. Cannot be used in combination with explicit FTP.`,
 
 When using explicit FTP over TLS the client explicitly requests
 security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTP.`,
+to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
@@ -315,18 +315,33 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+// Return a *textproto.Error if err contains one or nil otherwise
+func textprotoError(err error) (errX *textproto.Error) {
+	if errors.As(err, &errX) {
+		return errX
+	}
+	return nil
+}
+
+// returns true if this FTP error should be retried
+func isRetriableFtpError(err error) bool {
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
+			return true
+		}
+	}
+	return false
+}
+
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
+	if isRetriableFtpError(err) {
+		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -463,8 +478,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
-		var tpErr *textproto.Error
-		if !errors.As(err, &tpErr) {
+		if tpErr := textprotoError(err); tpErr != nil {
 			nopErr := c.NoOp()
 			if nopErr != nil {
 				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
@@ -613,8 +627,7 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
@@ -625,8 +638,7 @@ func translateErrorFile(err error) error {
 
 // translateErrorDir turns FTP errors into rclone errors if possible for a directory
 func translateErrorDir(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
@@ -657,8 +669,7 @@ func (f *Fs) dirFromStandardPath(dir string) string {
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	fullPath := path.Join(f.root, remote)
-	if fullPath == "" || fullPath == "." || fullPath == "/" {
+	if remote == "" || remote == "." || remote == "/" {
 		// if root, assume exists and synthesize an entry
 		return &ftp.Entry{
 			Name: "",
@@ -666,13 +677,32 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 			Time: time.Now(),
 		}, nil
 	}
-	dir := path.Dir(fullPath)
-	base := path.Base(fullPath)
 
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("findItem: %w", err)
 	}
+
+	// returns TRUE if MLST is supported which is required to call GetEntry
+	if c.IsTimePreciseInList() {
+		entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
+		f.putFtpConnection(&c, err)
+		if err != nil {
+			err = translateErrorFile(err)
+			if err == fs.ErrorObjectNotFound {
+				return nil, nil
+			}
+			return nil, err
+		}
+		if entry != nil {
+			f.entryToStandard(entry)
+		}
+		return entry, nil
+	}
+
+	dir := path.Dir(remote)
+	base := path.Base(remote)
+
 	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -691,7 +721,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return nil, err
 	}
@@ -713,7 +743,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 
 // dirExists checks the directory pointed to by remote exists or not
 func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return false, fmt.Errorf("dirExists: %w", err)
 	}
@@ -857,32 +887,18 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // getInfo reads the FileInfo for a path
 func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	dir := path.Dir(remote)
-	base := path.Base(remote)
-
-	c, err := f.getFtpConnection(ctx)
+	file, err := f.findItem(ctx, remote)
 	if err != nil {
-		return nil, fmt.Errorf("getInfo: %w", err)
-	}
-	files, err := c.List(f.dirFromStandardPath(dir))
-	f.putFtpConnection(&c, err)
-	if err != nil {
-		return nil, translateErrorFile(err)
-	}
-
-	for i := range files {
-		file := files[i]
-		f.entryToStandard(file)
-		if file.Name == base {
-			info := &FileInfo{
-				Name:    remote,
-				Size:    file.Size,
-				ModTime: file.Time,
-				precise: f.fLstTime,
-				IsDir:   file.Type == ftp.EntryTypeFolder,
-			}
-			return info, nil
+		return nil, err
+	} else if file != nil {
+		info := &FileInfo{
+			Name:    remote,
+			Size:    file.Size,
+			ModTime: file.Time,
+			precise: f.fLstTime,
+			IsDir:   file.Type == ftp.EntryTypeFolder,
 		}
+		return info, nil
 	}
 	return nil, fs.ErrorObjectNotFound
 }
@@ -913,8 +929,7 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	}
 	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
@@ -1155,8 +1170,7 @@ func (f *ftpReadCloser) Close() error {
 	// mask the error if it was caused by a premature close
 	// NB StatusAboutToSend is to work around a bug in pureftpd
 	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
@@ -1182,15 +1196,26 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+
+	var (
+		fd *ftp.Response
+		c  *ftp.ServerConn
+	)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		c, err = o.fs.getFtpConnection(ctx)
+		if err != nil {
+			return false, err // getFtpConnection has retries already
+		}
+		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+		if err != nil {
+			o.fs.putFtpConnection(&c, err)
+		}
+		return shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-	if err != nil {
-		o.fs.putFtpConnection(&c, err)
-		return nil, fmt.Errorf("open: %w", err)
-	}
+
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }
@@ -1223,13 +1248,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	// Ignore error 250 here - send by some servers
-	if err != nil {
-		switch errX := err.(type) {
-		case *textproto.Error:
-			switch errX.Code {
-			case ftp.StatusRequestedFileActionOK:
-				err = nil
-			}
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusRequestedFileActionOK:
+			err = nil
 		}
 	}
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,8 +19,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -82,7 +82,8 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			if saFile != "" || saCreds != "" || anonymous == "true" {
+			envAuth, _ := m.Get("env_auth")
+			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -92,6 +93,9 @@ func init() {
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "project_number",
 			Help: "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+		}, {
+			Name: "user_project",
+			Help: "User project.\n\nOptional - needed only for requester pays.",
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
@@ -330,6 +334,17 @@ can't check the size and hash but the file contents will be decompressed.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:    "env_auth",
+			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default: false,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 }
@@ -337,6 +352,7 @@ can't check the size and hash but the file contents will be decompressed.
 // Options defines the configuration for this backend
 type Options struct {
 	ProjectNumber             string               `config:"project_number"`
+	UserProject               string               `config:"user_project"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
 	Anonymous                 bool                 `config:"anonymous"`
@@ -349,6 +365,7 @@ type Options struct {
 	Decompress                bool                 `config:"decompress"`
 	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote storage server
@@ -487,7 +504,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -500,6 +517,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {
@@ -541,7 +563,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// Check to see if the object exists
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
+			get := f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx)
+			if f.opt.UserProject != "" {
+				get = get.UserProject(f.opt.UserProject)
+			}
+			_, err = get.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -601,6 +627,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		directory += "/"
 	}
 	list := f.svc.Objects.List(bucket).Prefix(directory).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		list = list.UserProject(f.opt.UserProject)
+	}
 	if !recurse {
 		list = list.Delimiter("/")
 	}
@@ -707,6 +736,9 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		return nil, errors.New("can't list buckets without project number")
 	}
 	listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		listBuckets = listBuckets.UserProject(f.opt.UserProject)
+	}
 	for {
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
@@ -836,7 +868,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
+			list := f.svc.Objects.List(bucket).MaxResults(1).Context(ctx)
+			if f.opt.UserProject != "" {
+				list = list.UserProject(f.opt.UserProject)
+			}
+			_, err = list.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -871,7 +907,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 			if !f.opt.BucketPolicyOnly {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
-			_, err = insertBucket.Context(ctx).Do()
+			insertBucket = insertBucket.Context(ctx)
+			if f.opt.UserProject != "" {
+				insertBucket = insertBucket.UserProject(f.opt.UserProject)
+			}
+			_, err = insertBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	}, nil)
@@ -896,7 +936,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	}
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
-			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
+			deleteBucket := f.svc.Buckets.Delete(bucket).Context(ctx)
+			if f.opt.UserProject != "" {
+				deleteBucket = deleteBucket.UserProject(f.opt.UserProject)
+			}
+			err = deleteBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	})
@@ -942,7 +986,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var rewriteResponse *storage.RewriteResponse
 	for {
 		err = f.pacer.Call(func() (bool, error) {
-			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
+			rewriteRequest = rewriteRequest.Context(ctx)
+			if f.opt.UserProject != "" {
+				rewriteRequest.UserProject(f.opt.UserProject)
+			}
+			rewriteResponse, err = rewriteRequest.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1053,7 +1101,11 @@ func (o *Object) setMetaData(info *storage.Object) {
 func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
+		get := o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			get = get.UserProject(o.fs.opt.UserProject)
+		}
+		object, err = get.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1125,7 +1177,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 		if !o.fs.opt.BucketPolicyOnly {
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = copyObject.Context(ctx).Do()
+		copyObject = copyObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			copyObject = copyObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = copyObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1142,6 +1198,9 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.fs.opt.UserProject != "" {
+		o.url = o.url + "&userProject=" + o.fs.opt.UserProject
+	}
 	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
 	if err != nil {
 		return nil, err
@@ -1234,7 +1293,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if !o.fs.opt.BucketPolicyOnly {
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = insertObject.Context(ctx).Do()
+		insertObject = insertObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			insertObject = insertObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = insertObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1249,7 +1312,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
+		deleteBucket := o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			deleteBucket = deleteBucket.UserProject(o.fs.opt.UserProject)
+		}
+		err = deleteBucket.Do()
 		return shouldRetry(ctx, err)
 	})
 	return err

backend/googlephotos/googlephotos_test.go

@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"testing"
@@ -12,7 +12,6 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -56,7 +55,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+			dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -99,7 +98,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := ioutil.ReadAll(in)
+				buf, err := io.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)
@@ -221,7 +220,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+		dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/hasher/commands.go

@@ -161,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
 			if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
 				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
 			}
-			accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
+			accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
 			doneCount++
 		}
 	})

backend/hasher/object.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"time"
 
@@ -118,7 +117,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(ioutil.Discard, r); err != nil {
+	if _, err = io.Copy(io.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}

backend/hdfs/hdfs.go

@@ -42,9 +42,9 @@ for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
 			Help: `Kerberos data transfer protection: authentication|integrity|privacy.
 
 Specifies whether or not authentication, data signature integrity
-checks, and wire encryption is required when communicating the the
-datanodes. Possible values are 'authentication', 'integrity' and
-'privacy'. Used only with KERBEROS enabled.`,
+checks, and wire encryption are required when communicating with
+the datanodes. Possible values are 'authentication', 'integrity'
+and 'privacy'. Used only with KERBEROS enabled.`,
 			Examples: []fs.OptionExample{{
 				Value: "privacy",
 				Help:  "Ensure authentication, integrity and encryption enabled.",

backend/hidrive/helpers.go

@@ -294,15 +294,6 @@ func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType Cop
 	return &result, nil
 }
 
-// copyDirectory moves the directory at the source-path to the destination-path and
-// returns the resulting api-object if successful.
-//
-// The operation will only be successful
-// if the parent-directory of the destination-path exists.
-func (f *Fs) copyDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
-	return f.copyOrMove(ctx, true, CopyOriginalPreserveModTime, source, destination, onExist)
-}
-
 // moveDirectory moves the directory at the source-path to the destination-path and
 // returns the resulting api-object if successful.
 //

backend/hidrive/hidrive.go

@@ -2,7 +2,7 @@
 package hidrive
 
 // FIXME HiDrive only supports file or folder names of 255 characters or less.
-// Operations that create files oder folder with longer names will throw a HTTP error:
+// Operations that create files or folders with longer names will throw an HTTP error:
 // - 422 Unprocessable Entity
 // A more graceful way for rclone to handle this may be desirable.
 
@@ -338,7 +338,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("could not access root-prefix: %w", err)
 		}
 		if item.Type != api.HiDriveObjectTypeDirectory {
-			return nil, errors.New("The root-prefix needs to point to a valid directory or be empty")
+			return nil, errors.New("the root-prefix needs to point to a valid directory or be empty")
 		}
 	}
 

backend/http/http.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -305,7 +304,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat(ctx)
+	err := o.head(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -317,15 +316,6 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-// parse s into an int64, on failure return def
-func parseInt64(s string, def int64) int64 {
-	n, e := strconv.ParseInt(s, 10, 64)
-	if e != nil {
-		return def
-	}
-	return n
-}
-
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -500,7 +490,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.stat(ctx); err {
+				switch err := file.head(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
@@ -579,8 +569,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// stat updates the info field in the Object
-func (o *Object) stat(ctx context.Context) error {
+// head sends a HEAD request to update info fields in the Object
+func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -601,13 +591,19 @@ func (o *Object) stat(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
+	return o.decodeMetadata(ctx, res)
+}
+
+// decodeMetadata updates info fields in the Object according to HTTP response headers
+func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	o.size = rest.ParseSizeFromHeaders(res.Header)
+
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -653,6 +649,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
+	if err = o.decodeMetadata(ctx, res); err != nil {
+		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
+	}
 	return res.Body, nil
 }
 

backend/http/http_internal_test.go

@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -33,20 +33,21 @@ var (
 	lineEndSize = 1
 )
 
-// prepareServer the test server and return a function to tidy it up afterwards
-func prepareServer(t *testing.T) (configmap.Simple, func()) {
+// prepareServer prepares the test server and shuts it down automatically
+// when the test completes.
+func prepareServer(t *testing.T) configmap.Simple {
 	// file server for test/files
 	fileServer := http.FileServer(http.Dir(filesPath))
 
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := ioutil.ReadDir(filesPath)
+	fileList, err := os.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -78,20 +79,21 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 		"url":     ts.URL,
 		"headers": strings.Join(headers, ","),
 	}
+	t.Cleanup(ts.Close)
 
-	// return a function to tidy up
-	return m, ts.Close
+	return m
 }
 
-// prepare the test server and return a function to tidy it up afterwards
-func prepare(t *testing.T) (fs.Fs, func()) {
-	m, tidy := prepareServer(t)
+// prepare prepares the test server and shuts it down automatically
+// when the test completes.
+func prepare(t *testing.T) fs.Fs {
+	m := prepareServer(t)
 
 	// Instantiate it
 	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
-	return f, tidy
+	return f
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
@@ -134,22 +136,19 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 }
 
 func TestListRoot(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 	testListRoot(t, f, false)
 }
 
 func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
+	f := prepare(t)
 	f.(*Fs).opt.NoSlash = true
-	defer tidy()
 
 	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
@@ -166,8 +165,7 @@ func TestListSubDir(t *testing.T) {
 }
 
 func TestNewObject(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -194,36 +192,69 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	m := prepareServer(t)
 
-	o, err := f.NewObject(context.Background(), "four/under four.txt")
-	require.NoError(t, err)
+	for _, head := range []bool{false, true} {
+		if !head {
+			m.Set("no_head", "true")
+		}
+		f, err := NewFs(context.Background(), remoteName, "", m)
+		require.NoError(t, err)
 
-	// Test normal read
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	if lineEndSize == 2 {
-		assert.Equal(t, "beetroot\r\n", string(data))
-	} else {
-		assert.Equal(t, "beetroot\n", string(data))
+		for _, rangeRead := range []bool{false, true} {
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			if !head {
+				// Test mod time is still indeterminate
+				tObj := o.ModTime(context.Background())
+				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
+
+				// Test file size is still indeterminate
+				assert.Equal(t, int64(-1), o.Size())
+			}
+
+			var data []byte
+			if !rangeRead {
+				// Test normal read
+				fd, err := o.Open(context.Background())
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				if lineEndSize == 2 {
+					assert.Equal(t, "beetroot\r\n", string(data))
+				} else {
+					assert.Equal(t, "beetroot\n", string(data))
+				}
+			} else {
+				// Test with range request
+				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				assert.Equal(t, "eetro", string(data))
+			}
+
+			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+			require.NoError(t, err)
+			tFile := fi.ModTime()
+
+			// Test the time is always correct on the object after file open
+			tObj := o.ModTime(context.Background())
+			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+			if !rangeRead {
+				// Test the file size
+				assert.Equal(t, int64(len(data)), o.Size())
+			}
+		}
 	}
-
-	// Test with range request
-	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-	require.NoError(t, err)
-	data, err = ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, "eetro", string(data))
 }
 
 func TestMimeType(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -234,8 +265,7 @@ func TestMimeType(t *testing.T) {
 }
 
 func TestIsAFileRoot(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
@@ -244,8 +274,7 @@ func TestIsAFileRoot(t *testing.T) {
 }
 
 func TestIsAFileSubDir(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)

backend/jottacloud/jottacloud.go

@@ -12,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -822,7 +821,7 @@ func (f *Fs) allocatePathRaw(file string, absolute bool) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
+		refreshBody, err := io.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -832,7 +831,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -1789,7 +1788,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = ioutil.TempFile("", cachePrefix)
+		tempFile, err = os.CreateTemp("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1817,7 +1816,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = ioutil.ReadAll(teeReader)
+		inData, err = io.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1839,12 +1838,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err == nil {
 			// if the object exists delete it
 			err = o.remove(ctx, true)
-			if err != nil {
+			if err != nil && err != fs.ErrorObjectNotFound {
+				// if delete failed then report that, unless it was because the file did not exist after all
 				return fmt.Errorf("failed to remove old object: %w", err)
 			}
-		}
-		// if the object does not exist we can just continue but if the error is something different we should report that
-		if err != fs.ErrorObjectNotFound {
+		} else if err != fs.ErrorObjectNotFound {
+			// if the object does not exist we can just continue but if the error is something different we should report that
 			return err
 		}
 	}
@@ -1914,7 +1913,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		_, err = io.CopyN(io.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}
@@ -1931,7 +1930,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.md5 = result.Md5
 		o.modTime = time.Unix(result.Modified/1000, 0)
 	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
+		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
 		return o.readMetaData(ctx, true)
 	}
 
@@ -1952,10 +1951,17 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 		opts.Parameters.Set("dl", "true")
 	}
 
-	return o.fs.pacer.Call(func() (bool, error) {
+	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
 		return shouldRetry(ctx, resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// attempting to hard delete will fail if path does not exist, but standard delete will succeed
+		if apiErr.StatusCode == http.StatusNotFound {
+			return fs.ErrorObjectNotFound
+		}
+	}
+	return err
 }
 
 // Remove an object

backend/local/local.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -124,8 +123,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy -
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
@@ -267,7 +266,10 @@ type Object struct {
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+var (
+	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
+)
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -311,7 +313,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
+	// Check to see if this is a .rclonelink if not found
+	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
+	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
+		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
+	}
 	if err == nil && f.isRegular(fi.Mode()) {
+		// Handle the odd case, that a symlink was specified by name without the link suffix
+		if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+			return nil, errLinksNeedsSuffix
+		}
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -504,7 +515,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 								continue
 							}
 						}
-						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
+						err = fmt.Errorf("failed to read directory %q: %w", namepath, fierr)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
@@ -521,15 +532,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			name := fi.Name()
 			mode := fi.Mode()
 			newRemote := f.cleanRemote(dir, name)
-			// Don't include non directory if not included
-			// we leave directory filtering to the layer above
-			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
-				continue
-			}
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
+				// Quietly skip errors on excluded files and directories
+				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
@@ -554,6 +564,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
+				// Don't include non directory if not included
+				// we leave directory filtering to the layer above
+				if useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err
@@ -646,7 +661,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := ioutil.TempFile("", "rclone")
+	fd, err := os.CreateTemp("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -1073,7 +1088,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read
@@ -1400,30 +1415,27 @@ func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
 }
 
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS == "windows" {
-		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
+	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
+		if !filepath.IsAbs(s) {
 			s2, err := filepath.Abs(s)
 			if err == nil {
 				s = s2
 			}
+		} else {
+			s = filepath.Clean(s)
 		}
+	}
+	if runtime.GOOS == "windows" {
 		s = filepath.ToSlash(s)
 		vol := filepath.VolumeName(s)
 		s = vol + enc.FromStandardPath(s[len(vol):])
 		s = filepath.FromSlash(s)
-
 		if !noUNC {
 			// Convert to UNC
 			s = file.UNCPath(s)
 		}
 		return s
 	}
-	if !filepath.IsAbs(s) {
-		s2, err := filepath.Abs(s)
-		if err == nil {
-			s = s2
-		}
-	}
 	s = enc.FromStandardPath(s)
 	return s
 }

backend/local/local_internal_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
@@ -33,7 +34,6 @@ func TestMain(m *testing.M) {
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
@@ -78,7 +78,6 @@ func TestUpdatingCheck(t *testing.T) {
 func TestSymlink(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	f := r.Flocal.(*Fs)
 	dir := f.root
 
@@ -147,10 +146,24 @@ func TestSymlink(t *testing.T) {
 	_, err = r.Flocal.NewObject(ctx, "symlink2.txt")
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
+	// Check that NewFs works with the suffixed version and --links
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.Equal(t, dir, f2.(*Fs).root)
+
+	// Check that NewFs doesn't see the non suffixed version with --links
+	f2, err = NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, errLinksNeedsSuffix, err)
+	require.Nil(t, f2)
+
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
+	contents, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -158,7 +171,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
+	contents, err = io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
@@ -177,7 +190,6 @@ func TestSymlinkError(t *testing.T) {
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -208,7 +220,6 @@ func TestHashOnUpdate(t *testing.T) {
 func TestHashOnDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -237,7 +248,6 @@ func TestHashOnDelete(t *testing.T) {
 func TestMetadata(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "metafile.txt"
 	when := time.Now()
 	const dayLength = len("2001-01-01")
@@ -372,7 +382,6 @@ func TestMetadata(t *testing.T) {
 func TestFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	when := time.Now()
 	r.WriteFile("included", "included file", when)
 	r.WriteFile("excluded", "excluded file", when)
@@ -401,3 +410,107 @@ func TestFilter(t *testing.T) {
 	sort.Sort(entries)
 	require.Equal(t, "[included]", fmt.Sprint(entries))
 }
+
+func testFilterSymlink(t *testing.T, copyLinks bool) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
+	r.WriteFile("included.file", "included file", when)
+	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
+	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
+	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
+	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
+
+	defer func() {
+		// Reset -L/-l mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Lstat
+	}()
+	if copyLinks {
+		// Set fs into "-L" mode
+		f.opt.FollowSymlinks = true
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Stat
+	} else {
+		// Set fs into "-l" mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = true
+		f.lstat = os.Lstat
+	}
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Reset global error count
+	accounting.Stats(ctx).ResetErrors()
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included.file"))
+	require.NoError(t, fi.AddRule("+ included.dir/**"))
+	if copyLinks {
+		require.NoError(t, fi.AddRule("+ included.file.link"))
+		require.NoError(t, fi.AddRule("+ included.dir.link/**"))
+	} else {
+		require.NoError(t, fi.AddRule("+ included.file.link.rclonelink"))
+		require.NoError(t, fi.AddRule("+ included.dir.link.rclonelink"))
+	}
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+
+	if copyLinks {
+		// Check 1 global errors one for each dangling symlink
+		assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+	} else {
+		// Check 0 global errors as dangling symlink copied properly
+		assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+	}
+	accounting.Stats(ctx).ResetErrors()
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[dangling.link.rclonelink included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Check listing through a symlink still works
+	entries, err = f.List(ctx, "included.dir")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
+}
+
+func TestFilterSymlinkCopyLinks(t *testing.T) {
+	testFilterSymlink(t, true)
+}
+
+func TestFilterSymlinkLinks(t *testing.T) {
+	testFilterSymlink(t, false)
+}

backend/local/metadata_linux.go

@@ -91,7 +91,7 @@ func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
 		// The types of t.Sec and t.Nsec vary from int32 to int64 on
 		// different Linux architectures so we need to cast them to
 		// int64 here and hence need to quiet the linter about
-		// unecessary casts.
+		// unnecessary casts.
 		//
 		// nolint: unconvert
 		m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat))

backend/local/remove_test.go

@@ -1,7 +1,6 @@
 package local
 
 import (
-	"io/ioutil"
 	"os"
 	"sync"
 	"testing"
@@ -13,7 +12,7 @@ import (
 
 // Check we can remove an open file
 func TestRemove(t *testing.T) {
-	fd, err := ioutil.TempFile("", "rclone-remove-test")
+	fd, err := os.CreateTemp("", "rclone-remove-test")
 	require.NoError(t, err)
 	name := fd.Name()
 	defer func() {

backend/mailru/api/helpers.go

@@ -69,6 +69,11 @@ func (w *BinWriter) WritePu64(val int64) {
 	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
 }
 
+// WriteP64 writes an signed long as unsigned varint
+func (w *BinWriter) WriteP64(val int64) {
+	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
+}
+
 // WriteString writes a zero-terminated string
 func (w *BinWriter) WriteString(str string) {
 	buf := []byte(str)

backend/mailru/mailru.go

@@ -18,7 +18,6 @@ import (
 
 	"encoding/hex"
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -91,8 +90,13 @@ func init() {
 			Help:     "User name (usually email).",
 			Required: true,
 		}, {
-			Name:       "pass",
-			Help:       "Password.",
+			Name: "pass",
+			Help: `Password.
+
+This must be an app password - rclone will not work with your normal
+password. See the Configuration section in the docs for how to make an
+app password.
+`,
 			Required:   true,
 			IsPassword: true,
 		}, {
@@ -641,12 +645,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
 		return nil, -1, err
 	}
 
-	mTime := int64(item.Mtime)
-	if mTime < 0 {
-		fs.Debugf(f, "Fixing invalid timestamp %d on mailru file %q", mTime, remote)
-		mTime = 0
-	}
-	modTime := time.Unix(mTime, 0)
+	modTime := time.Unix(int64(item.Mtime), 0)
 
 	isDir, err := f.isDir(item.Kind, remote)
 	if err != nil {
@@ -1660,7 +1659,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Attempt to put by calculating hash in memory
 	if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
-		fileBuf, err = ioutil.ReadAll(in)
+		fileBuf, err = io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1703,7 +1702,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if size <= mrhash.Size {
 		// Optimize upload: skip extra request if data fits in the hash buffer.
 		if fileBuf == nil {
-			fileBuf, err = ioutil.ReadAll(wrapIn)
+			fileBuf, err = io.ReadAll(wrapIn)
 		}
 		if fileHash == nil && err == nil {
 			fileHash = mrhash.Sum(fileBuf)
@@ -2058,7 +2057,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	req.WritePu16(0) // revision
 	req.WriteString(o.fs.opt.Enc.FromStandardPath(o.absPath()))
 	req.WritePu64(o.size)
-	req.WritePu64(o.modTime.Unix())
+	req.WriteP64(o.modTime.Unix())
 	req.WritePu32(0)
 	req.Write(o.mrHash)
 
@@ -2214,7 +2213,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		fs.Debugf(o, "Server returned full content instead of range")
 		if start > 0 {
 			// Discard the beginning of the data
-			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
+			_, err = io.CopyN(io.Discard, wrapStream, start)
 			if err != nil {
 				closeBody(res)
 				return nil, err

backend/mega/mega.go

@@ -83,6 +83,17 @@ than permanently deleting them.  If you specify this then rclone will
 permanently delete objects instead.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "use_https",
+			Help: `Use HTTPS for transfers.
+
+MEGA uses plain text HTTP connections by default.
+Some ISPs throttle HTTP connections, this causes transfers to become very slow.
+Enabling this will force MEGA to use HTTPS for all transfers.
+HTTPS is normally not necessary since all data is already encrypted anyway.
+Enabling it will increase CPU usage and add network overhead.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -100,6 +111,7 @@ type Options struct {
 	Pass       string               `config:"pass"`
 	Debug      bool                 `config:"debug"`
 	HardDelete bool                 `config:"hard_delete"`
+	UseHTTPS   bool                 `config:"use_https"`
 	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -204,6 +216,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if srv == nil {
 		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
+		srv.SetHTTPS(opt.UseHTTPS)
 		srv.SetLogger(func(format string, v ...interface{}) {
 			fs.Infof("*go-mega*", format, v...)
 		})

backend/memory/memory.go

@@ -8,7 +8,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"strings"
 	"sync"
@@ -575,7 +574,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		data = data[:limit]
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
+	return io.NopCloser(bytes.NewBuffer(data)), nil
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -583,7 +582,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := ioutil.ReadAll(in)
+	data, err := io.ReadAll(in)
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}

backend/netstorage/netstorage.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -820,6 +819,8 @@ func (f *Fs) getAuth(req *http.Request) error {
 	// Set Authorization header
 	dataHeader := generateDataHeader(f)
 	path := req.URL.RequestURI()
+	//lint:ignore SA1008 false positive when running staticcheck, the header name is according to docs even if not canonical
+	//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1008
 	actionHeader := req.Header["X-Akamai-ACS-Action"][0]
 	fs.Debugf(nil, "NetStorage API %s call %s for path %q", req.Method, actionHeader, path)
 	req.Header.Set("X-Akamai-ACS-Auth-Data", dataHeader)
@@ -972,7 +973,7 @@ func (o *Object) netStorageUploadRequest(ctx context.Context, in io.Reader, src
 		URL = o.fs.url(src.Remote())
 	}
 	if strings.HasSuffix(URL, ".rclonelink") {
-		bits, err := ioutil.ReadAll(in)
+		bits, err := io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1058,7 +1059,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 	if strings.HasSuffix(URL, ".rclonelink") && o.target != "" {
 		fs.Infof(nil, "Converting a symlink to the rclonelink file on download %q", URL)
 		reader := strings.NewReader(o.target)
-		readcloser := ioutil.NopCloser(reader)
+		readcloser := io.NopCloser(reader)
 		return readcloser, nil
 	}
 

backend/onedrive/api/types.go

@@ -126,6 +126,7 @@ type HashesType struct {
 	Sha1Hash     string `json:"sha1Hash"`     // hex encoded SHA1 hash for the contents of the file (if available)
 	Crc32Hash    string `json:"crc32Hash"`    // hex encoded CRC32 value of the file (if available)
 	QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available)
+	Sha256Hash   string `json:"sha256Hash"`   // hex encoded SHA256 value of the file (if available)
 }
 
 // FileFacet groups file-related data on OneDrive into a single structure.

backend/onedrive/onedrive.go

@@ -259,6 +259,48 @@ this flag there.
 At the time of writing this only works with OneDrive personal paid accounts.
 `,
 			Advanced: true,
+		}, {
+			Name:    "hash_type",
+			Default: "auto",
+			Help: `Specify the hash in use for the backend.
+
+This specifies the hash type in use. If set to "auto" it will use the
+default hash which is QuickXorHash.
+
+Before rclone 1.62 an SHA1 hash was used by default for Onedrive
+Personal. For 1.62 and later the default is to use a QuickXorHash for
+all onedrive types. If an SHA1 hash is desired then set this option
+accordingly.
+
+From July 2023 QuickXorHash will be the only available hash for
+both OneDrive for Business and OneDriver Personal.
+
+This can be set to "none" to not use any hashes.
+
+If the hash requested does not exist on the object, it will be
+returned as an empty string which is treated as a missing hash by
+rclone.
+`,
+			Examples: []fs.OptionExample{{
+				Value: "auto",
+				Help:  "Rclone chooses the best hash",
+			}, {
+				Value: "quickxor",
+				Help:  "QuickXor",
+			}, {
+				Value: "sha1",
+				Help:  "SHA1",
+			}, {
+				Value: "sha256",
+				Help:  "SHA256",
+			}, {
+				Value: "crc32",
+				Help:  "CRC32",
+			}, {
+				Value: "none",
+				Help:  "None - don't use any hashes",
+			}},
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -511,7 +553,7 @@ Example: "https://contoso.sharepoint.com/sites/mysite" or "mysite"
 `)
 	case "url_end":
 		siteURL := config.Result
-		re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
+		re := regexp.MustCompile(`https://.*\.sharepoint\.com/sites/(.*)`)
 		match := re.FindStringSubmatch(siteURL)
 		if len(match) == 2 {
 			return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
@@ -597,6 +639,7 @@ type Options struct {
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
 	LinkPassword            string               `config:"link_password"`
+	HashType                string               `config:"hash_type"`
 	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -613,6 +656,7 @@ type Fs struct {
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
+	hashType     hash.Type          // type of the hash we are using
 }
 
 // Object describes a OneDrive object
@@ -626,8 +670,7 @@ type Object struct {
 	size          int64     // size of the object
 	modTime       time.Time // modification time of the object
 	id            string    // ID of the object
-	sha1          string    // SHA-1 of the object content
-	quickxorhash  string    // QuickXorHash of the object content
+	hash          string    // Hash of the content, usually QuickXorHash but set as hash_type
 	mimeType      string    // Content-Type of object from server (may not be as uploaded)
 }
 
@@ -882,6 +925,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
 		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		hashType:  QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -891,6 +935,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
+	// Set the user defined hash
+	if opt.HashType == "auto" || opt.HashType == "" {
+		opt.HashType = QuickXorHashType.String()
+	}
+	err = f.hashType.Set(opt.HashType)
+	if err != nil {
+		return nil, err
+	}
+
 	// Disable change polling in China region
 	// See: https://github.com/rclone/rclone/issues/6444
 	if f.opt.Region == regionCN {
@@ -1556,10 +1609,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	if f.driveType == driveTypePersonal {
-		return hash.Set(hash.SHA1)
-	}
-	return hash.Set(QuickXorHashType)
+	return hash.Set(f.hashType)
 }
 
 // PublicLink returns a link for downloading without account.
@@ -1674,6 +1724,10 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	token := make(chan struct{}, f.ci.Checkers)
 	var wg sync.WaitGroup
 	err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+		if err != nil {
+			fs.Errorf(f, "Failed to list %q: %v", path, err)
+			return nil
+		}
 		err = entries.ForObjectError(func(obj fs.Object) error {
 			o, ok := obj.(*Object)
 			if !ok {
@@ -1768,14 +1822,8 @@ func (o *Object) rootPath() string {
 
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if o.fs.driveType == driveTypePersonal {
-		if t == hash.SHA1 {
-			return o.sha1, nil
-		}
-	} else {
-		if t == QuickXorHashType {
-			return o.quickxorhash, nil
-		}
+	if t == o.fs.hashType {
+		return o.hash, nil
 	}
 	return "", hash.ErrUnsupported
 }
@@ -1806,16 +1854,23 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	file := info.GetFile()
 	if file != nil {
 		o.mimeType = file.MimeType
-		if file.Hashes.Sha1Hash != "" {
-			o.sha1 = strings.ToLower(file.Hashes.Sha1Hash)
-		}
-		if file.Hashes.QuickXorHash != "" {
-			h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
-			if err != nil {
-				fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
-			} else {
-				o.quickxorhash = hex.EncodeToString(h)
+		o.hash = ""
+		switch o.fs.hashType {
+		case QuickXorHashType:
+			if file.Hashes.QuickXorHash != "" {
+				h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
+				if err != nil {
+					fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
+				} else {
+					o.hash = hex.EncodeToString(h)
+				}
 			}
+		case hash.SHA1:
+			o.hash = strings.ToLower(file.Hashes.Sha1Hash)
+		case hash.SHA256:
+			o.hash = strings.ToLower(file.Hashes.Sha256Hash)
+		case hash.CRC32:
+			o.hash = strings.ToLower(file.Hashes.Crc32Hash)
 		}
 	}
 	fileSystemInfo := info.GetFileSystemInfo()

backend/onedrive/quickxorhash/quickxorhash.go

@@ -7,51 +7,40 @@
 // See: https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
 package quickxorhash
 
-// This code was ported from the code snippet linked from
-// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
-// Which has the copyright
+// This code was ported from a fast C-implementation from
+// https://github.com/namazso/QuickXorHash
+// which has licenced as BSD Zero Clause License
+//
+// BSD Zero Clause License
+//
+// Copyright (c) 2022 namazso <admin@namazso.eu>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
 
-// ------------------------------------------------------------------------------
-//  Copyright (c) 2016 Microsoft Corporation
-//
-//  Permission is hereby granted, free of charge, to any person obtaining a copy
-//  of this software and associated documentation files (the "Software"), to deal
-//  in the Software without restriction, including without limitation the rights
-//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-//  copies of the Software, and to permit persons to whom the Software is
-//  furnished to do so, subject to the following conditions:
-//
-//  The above copyright notice and this permission notice shall be included in
-//  all copies or substantial portions of the Software.
-//
-//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-//  THE SOFTWARE.
-// ------------------------------------------------------------------------------
-
-import (
-	"hash"
-)
+import "hash"
 
 const (
 	// BlockSize is the preferred size for hashing
 	BlockSize = 64
 	// Size of the output checksum
-	Size           = 20
-	bitsInLastCell = 32
-	shift          = 11
-	widthInBits    = 8 * Size
-	dataSize       = (widthInBits-1)/64 + 1
+	Size        = 20
+	shift       = 11
+	widthInBits = 8 * Size
+	dataSize    = shift * widthInBits
 )
 
 type quickXorHash struct {
-	data        [dataSize]uint64
-	lengthSoFar uint64
-	shiftSoFar  int
+	data [dataSize]byte
+	size uint64
 }
 
 // New returns a new hash.Hash computing the quickXorHash checksum.
@@ -70,94 +59,37 @@ func New() hash.Hash {
 //
 // Implementations must not retain p.
 func (q *quickXorHash) Write(p []byte) (n int, err error) {
-	currentshift := q.shiftSoFar
-
-	// The bitvector where we'll start xoring
-	vectorArrayIndex := currentshift / 64
-
-	// The position within the bit vector at which we begin xoring
-	vectorOffset := currentshift % 64
-	iterations := len(p)
-	if iterations > widthInBits {
-		iterations = widthInBits
+	var i int
+	// fill last remain
+	lastRemain := q.size % dataSize
+	if lastRemain != 0 {
+		i += xorBytes(q.data[lastRemain:], p)
 	}
 
-	for i := 0; i < iterations; i++ {
-		isLastCell := vectorArrayIndex == len(q.data)-1
-		var bitsInVectorCell int
-		if isLastCell {
-			bitsInVectorCell = bitsInLastCell
-		} else {
-			bitsInVectorCell = 64
-		}
-
-		// There's at least 2 bitvectors before we reach the end of the array
-		if vectorOffset <= bitsInVectorCell-8 {
-			for j := i; j < len(p); j += widthInBits {
-				q.data[vectorArrayIndex] ^= uint64(p[j]) << uint(vectorOffset)
-			}
-		} else {
-			index1 := vectorArrayIndex
-			var index2 int
-			if isLastCell {
-				index2 = 0
-			} else {
-				index2 = vectorArrayIndex + 1
-			}
-			low := byte(bitsInVectorCell - vectorOffset)
-
-			xoredByte := byte(0)
-			for j := i; j < len(p); j += widthInBits {
-				xoredByte ^= p[j]
-			}
-			q.data[index1] ^= uint64(xoredByte) << uint(vectorOffset)
-			q.data[index2] ^= uint64(xoredByte) >> low
-		}
-		vectorOffset += shift
-		for vectorOffset >= bitsInVectorCell {
-			if isLastCell {
-				vectorArrayIndex = 0
-			} else {
-				vectorArrayIndex = vectorArrayIndex + 1
-			}
-			vectorOffset -= bitsInVectorCell
+	if i != len(p) {
+		for len(p)-i >= dataSize {
+			i += xorBytes(q.data[:], p[i:])
 		}
+		xorBytes(q.data[:], p[i:])
 	}
-
-	// Update the starting position in a circular shift pattern
-	q.shiftSoFar = (q.shiftSoFar + shift*(len(p)%widthInBits)) % widthInBits
-
-	q.lengthSoFar += uint64(len(p))
-
+	q.size += uint64(len(p))
 	return len(p), nil
 }
 
 // Calculate the current checksum
-func (q *quickXorHash) checkSum() (h [Size]byte) {
-	// Output the data as little endian bytes
-	ph := 0
-	for i := 0; i < len(q.data)-1; i++ {
-		d := q.data[i]
-		_ = h[ph+7] // bounds check
-		h[ph+0] = byte(d >> (8 * 0))
-		h[ph+1] = byte(d >> (8 * 1))
-		h[ph+2] = byte(d >> (8 * 2))
-		h[ph+3] = byte(d >> (8 * 3))
-		h[ph+4] = byte(d >> (8 * 4))
-		h[ph+5] = byte(d >> (8 * 5))
-		h[ph+6] = byte(d >> (8 * 6))
-		h[ph+7] = byte(d >> (8 * 7))
-		ph += 8
+func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
+	for i := 0; i < dataSize; i++ {
+		shift := (i * 11) % 160
+		shiftBytes := shift / 8
+		shiftBits := shift % 8
+		shifted := int(q.data[i]) << shiftBits
+		h[shiftBytes] ^= byte(shifted)
+		h[shiftBytes+1] ^= byte(shifted >> 8)
 	}
-	// remaining 32 bits
-	d := q.data[len(q.data)-1]
-	h[Size-4] = byte(d >> (8 * 0))
-	h[Size-3] = byte(d >> (8 * 1))
-	h[Size-2] = byte(d >> (8 * 2))
-	h[Size-1] = byte(d >> (8 * 3))
+	h[0] ^= h[20]
 
 	// XOR the file length with the least significant bits in little endian format
-	d = q.lengthSoFar
+	d := q.size
 	h[Size-8] ^= byte(d >> (8 * 0))
 	h[Size-7] ^= byte(d >> (8 * 1))
 	h[Size-6] ^= byte(d >> (8 * 2))
@@ -174,7 +106,7 @@ func (q *quickXorHash) checkSum() (h [Size]byte) {
 // It does not change the underlying hash state.
 func (q *quickXorHash) Sum(b []byte) []byte {
 	hash := q.checkSum()
-	return append(b, hash[:]...)
+	return append(b, hash[:Size]...)
 }
 
 // Reset resets the Hash to its initial state.
@@ -196,8 +128,10 @@ func (q *quickXorHash) BlockSize() int {
 }
 
 // Sum returns the quickXorHash checksum of the data.
-func Sum(data []byte) [Size]byte {
+func Sum(data []byte) (h [Size]byte) {
 	var d quickXorHash
 	_, _ = d.Write(data)
-	return d.checkSum()
+	s := d.checkSum()
+	copy(h[:], s[:])
+	return h
 }

backend/onedrive/quickxorhash/quickxorhash_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"hash"
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -166,3 +167,16 @@ func TestReset(t *testing.T) {
 
 // check interface
 var _ hash.Hash = (*quickXorHash)(nil)
+
+func BenchmarkQuickXorHash(b *testing.B) {
+	b.SetBytes(1 << 20)
+	buf := make([]byte, 1<<20)
+	rand.Read(buf)
+	h := New()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		h.Reset()
+		h.Write(buf)
+		h.Sum(nil)
+	}
+}

backend/onedrive/quickxorhash/xor.go

@@ -0,0 +1,20 @@
+//go:build !go1.20
+
+package quickxorhash
+
+func xorBytes(dst, src []byte) int {
+	n := len(dst)
+	if len(src) < n {
+		n = len(src)
+	}
+	if n == 0 {
+		return 0
+	}
+	dst = dst[:n]
+	//src = src[:n]
+	src = src[:len(dst)] // remove bounds check in loop
+	for i := range dst {
+		dst[i] ^= src[i]
+	}
+	return n
+}

backend/onedrive/quickxorhash/xor_1.20.go

@@ -0,0 +1,9 @@
+//go:build go1.20
+
+package quickxorhash
+
+import "crypto/subtle"
+
+func xorBytes(dst, src []byte) int {
+	return subtle.XORBytes(dst, src, dst)
+}

backend/oracleobjectstorage/byok.go

@@ -0,0 +1,145 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage/transfer"
+)
+
+const (
+	sseDefaultAlgorithm = "AES256"
+)
+
+func getSha256(p []byte) []byte {
+	h := sha256.New()
+	h.Write(p)
+	return h.Sum(nil)
+}
+
+func validateSSECustomerKeyOptions(opt *Options) error {
+	if opt.SSEKMSKeyID != "" && (opt.SSECustomerKeyFile != "" || opt.SSECustomerKey != "") {
+		return errors.New("oos: can't use vault sse_kms_key_id and local sse_customer_key at the same time")
+	}
+	if opt.SSECustomerKey != "" && opt.SSECustomerKeyFile != "" {
+		return errors.New("oos: can't use sse_customer_key and sse_customer_key_file at the same time")
+	}
+	if opt.SSEKMSKeyID != "" {
+		return nil
+	}
+	err := populateSSECustomerKeys(opt)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func populateSSECustomerKeys(opt *Options) error {
+	if opt.SSECustomerKeyFile != "" {
+		// Reads the base64-encoded AES key data from the specified file and computes its SHA256 checksum
+		data, err := os.ReadFile(expandPath(opt.SSECustomerKeyFile))
+		if err != nil {
+			return fmt.Errorf("oos: error reading sse_customer_key_file: %v", err)
+		}
+		opt.SSECustomerKey = strings.TrimSpace(string(data))
+	}
+	if opt.SSECustomerKey != "" {
+		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKey)
+		if err != nil {
+			return fmt.Errorf("oos: Could not decode sse_customer_key_file: %w", err)
+		}
+		sha256Checksum := base64.StdEncoding.EncodeToString(getSha256(decoded))
+		if opt.SSECustomerKeySha256 == "" {
+			opt.SSECustomerKeySha256 = sha256Checksum
+		} else {
+			if opt.SSECustomerKeySha256 != sha256Checksum {
+				return fmt.Errorf("the computed SHA256 checksum "+
+					"(%v) of the key doesn't match the config entry sse_customer_key_sha256=(%v)",
+					sha256Checksum, opt.SSECustomerKeySha256)
+			}
+		}
+		if opt.SSECustomerAlgorithm == "" {
+			opt.SSECustomerAlgorithm = sseDefaultAlgorithm
+		}
+	}
+	return nil
+}
+
+// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingyourencryptionkeys.htm
+func useBYOKPutObject(fs *Fs, request *objectstorage.PutObjectRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKHeadObject(fs *Fs, request *objectstorage.HeadObjectRequest) {
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKGetObject(fs *Fs, request *objectstorage.GetObjectRequest) {
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}
+
+func useBYOKUpload(fs *Fs, request *transfer.UploadRequest) {
+	if fs.opt.SSEKMSKeyID != "" {
+		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
+	}
+	if fs.opt.SSECustomerAlgorithm != "" {
+		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+	}
+	if fs.opt.SSECustomerKey != "" {
+		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+	}
+	if fs.opt.SSECustomerKeySha256 != "" {
+		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+	}
+}

backend/oracleobjectstorage/client.go

@@ -9,6 +9,8 @@ import (
 	"errors"
 	"net/http"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
 	"github.com/oracle/oci-go-sdk/v65/common/auth"
@@ -18,15 +20,33 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 )
 
+func expandPath(filepath string) (expandedPath string) {
+	if filepath == "" {
+		return filepath
+	}
+	cleanedPath := path.Clean(filepath)
+	expandedPath = cleanedPath
+	if strings.HasPrefix(cleanedPath, "~") {
+		rest := cleanedPath[2:]
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return expandedPath
+		}
+		expandedPath = path.Join(home, rest)
+	}
+	return
+}
+
 func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) {
 	switch opt.Provider {
 	case instancePrincipal:
 		return auth.InstancePrincipalConfigurationProvider()
 	case userPrincipal:
-		if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) {
-			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile)
+		expandConfigFilePath := expandPath(opt.ConfigFile)
+		if expandConfigFilePath != "" && !fileExists(expandConfigFilePath) {
+			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", expandConfigFilePath)
 		}
-		return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil
+		return common.CustomProfileConfigProvider(expandConfigFilePath, opt.ConfigProfile), nil
 	case resourcePrincipal:
 		return auth.ResourcePrincipalConfigurationProvider()
 	case noAuth:

backend/oracleobjectstorage/command.go

@@ -65,8 +65,8 @@ a bucket or with a bucket and path.
 	Long: `This command removes unfinished multipart uploads of age greater than
 max-age which defaults to 24 hours.
 
-Note that you can use -i/--dry-run with this command to see what it
-would do.
+Note that you can use --interactive/-i or --dry-run with this command to see what
+it would do.
 
     rclone backend cleanup oos:bucket/path/to/object
     rclone backend cleanup -o max-age=7w oos:bucket/path/to/object

backend/oracleobjectstorage/copy.go

@@ -74,6 +74,7 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err erro
 		BucketName:        common.String(srcBucket),
 		CopyObjectDetails: copyObjectDetails,
 	}
+	useBYOKCopyObject(f, &req)
 	var resp objectstorage.CopyObjectResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CopyObject(ctx, req)

backend/oracleobjectstorage/object.go

@@ -87,6 +87,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 		BucketName:    common.String(bucketName),
 		ObjectName:    common.String(objectPath),
 	}
+	useBYOKHeadObject(o.fs, &req)
 	var response objectstorage.HeadObjectResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -99,6 +100,7 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 				return nil, fs.ErrorObjectNotFound
 			}
 		}
+		fs.Errorf(o, "Failed to head object: %v", err)
 		return nil, err
 	}
 	o.fs.cache.MarkOK(bucketName)
@@ -331,7 +333,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 		ObjectName:    common.String(bucketPath),
 	}
 	o.applyGetObjectOptions(&req, options...)
-
+	useBYOKGetObject(o.fs, &req)
 	var resp objectstorage.GetObjectResponse
 	err := o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -433,6 +435,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			uploadRequest.StorageTier = storageTier
 		}
 		o.applyMultiPutOptions(&uploadRequest, options...)
+		useBYOKUpload(o.fs, &uploadRequest)
 		uploadStreamRequest := transfer.UploadStreamRequest{
 			UploadRequest: uploadRequest,
 			StreamReader:  in,
@@ -506,8 +509,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			req.StorageTier = storageTier
 		}
 		o.applyPutOptions(&req, options...)
+		useBYOKPutObject(o.fs, &req)
+		var resp objectstorage.PutObjectResponse
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err := o.fs.srv.PutObject(ctx, req)
+			resp, err = o.fs.srv.PutObject(ctx, req)
 			return shouldRetry(ctx, resp.HTTPResponse(), err)
 		})
 		if err != nil {

backend/oracleobjectstorage/options.go

@@ -17,9 +17,7 @@ const (
 	defaultUploadCutoff        = fs.SizeSuffix(200 * 1024 * 1024)
 	defaultUploadConcurrency   = 10
 	maxUploadCutoff            = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
-	minSleep                   = 100 * time.Millisecond
-	maxSleep                   = 5 * time.Minute
-	decayConstant              = 1 // bigger for slower decay, exponential
+	minSleep                   = 10 * time.Millisecond
 	defaultCopyTimeoutDuration = fs.Duration(time.Minute)
 )
 
@@ -47,23 +45,28 @@ https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfromins
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider          string               `config:"provider"`
-	Compartment       string               `config:"compartment"`
-	Namespace         string               `config:"namespace"`
-	Region            string               `config:"region"`
-	Endpoint          string               `config:"endpoint"`
-	Enc               encoder.MultiEncoder `config:"encoding"`
-	ConfigFile        string               `config:"config_file"`
-	ConfigProfile     string               `config:"config_profile"`
-	UploadCutoff      fs.SizeSuffix        `config:"upload_cutoff"`
-	ChunkSize         fs.SizeSuffix        `config:"chunk_size"`
-	UploadConcurrency int                  `config:"upload_concurrency"`
-	DisableChecksum   bool                 `config:"disable_checksum"`
-	CopyCutoff        fs.SizeSuffix        `config:"copy_cutoff"`
-	CopyTimeout       fs.Duration          `config:"copy_timeout"`
-	StorageTier       string               `config:"storage_tier"`
-	LeavePartsOnError bool                 `config:"leave_parts_on_error"`
-	NoCheckBucket     bool                 `config:"no_check_bucket"`
+	Provider             string               `config:"provider"`
+	Compartment          string               `config:"compartment"`
+	Namespace            string               `config:"namespace"`
+	Region               string               `config:"region"`
+	Endpoint             string               `config:"endpoint"`
+	Enc                  encoder.MultiEncoder `config:"encoding"`
+	ConfigFile           string               `config:"config_file"`
+	ConfigProfile        string               `config:"config_profile"`
+	UploadCutoff         fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency    int                  `config:"upload_concurrency"`
+	DisableChecksum      bool                 `config:"disable_checksum"`
+	CopyCutoff           fs.SizeSuffix        `config:"copy_cutoff"`
+	CopyTimeout          fs.Duration          `config:"copy_timeout"`
+	StorageTier          string               `config:"storage_tier"`
+	LeavePartsOnError    bool                 `config:"leave_parts_on_error"`
+	NoCheckBucket        bool                 `config:"no_check_bucket"`
+	SSEKMSKeyID          string               `config:"sse_kms_key_id"`
+	SSECustomerAlgorithm string               `config:"sse_customer_algorithm"`
+	SSECustomerKey       string               `config:"sse_customer_key"`
+	SSECustomerKeyFile   string               `config:"sse_customer_key_file"`
+	SSECustomerKeySha256 string               `config:"sse_customer_key_sha256"`
 }
 
 func newOptions() []fs.Option {
@@ -123,6 +126,22 @@ func newOptions() []fs.Option {
 			Value: "Default",
 			Help:  "Use the default profile",
 		}},
+	}, {
+		// Mapping from here: https://github.com/oracle/oci-go-sdk/blob/master/objectstorage/storage_tier.go
+		Name:     "storage_tier",
+		Help:     "The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm",
+		Default:  "Standard",
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "Standard",
+			Help:  "Standard storage tier, this is the default tier",
+		}, {
+			Value: "InfrequentAccess",
+			Help:  "InfrequentAccess storage tier",
+		}, {
+			Value: "Archive",
+			Help:  "Archive storage tier",
+		}},
 	}, {
 		Name: "upload_cutoff",
 		Help: `Cutoff for switching to chunked upload.
@@ -238,5 +257,59 @@ creation permissions.
 `,
 		Default:  false,
 		Advanced: true,
+	}, {
+		Name: "sse_customer_key_file",
+		Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated
+with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_key",
+		Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to
+encrypt or  decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is
+needed. For more information, see Using Your Own Keys for Server-Side Encryption 
+(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_key_sha256",
+		Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption
+key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 
+Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_kms_key_id",
+		Help: `if using your own master key in vault, this header specifies the
+OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
+the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
+Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}},
+	}, {
+		Name: "sse_customer_algorithm",
+		Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm.
+Object Storage supports "AES256" as the encryption algorithm. For more information, see
+Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
+		Advanced: true,
+		Examples: []fs.OptionExample{{
+			Value: "",
+			Help:  "None",
+		}, {
+			Value: sseDefaultAlgorithm,
+			Help:  sseDefaultAlgorithm,
+		}},
 	}}
 }

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -59,19 +59,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
+	err = validateSSECustomerKeyOptions(opt)
+	if err != nil {
+		return nil, err
+	}
 	ci := fs.GetConfig(ctx)
 	objectStorageClient, err := newObjectStorageClient(ctx, opt)
 	if err != nil {
 		return nil, err
 	}
-	p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))
+	pc := fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep)))
+	// Set pacer retries to 2 (1 try and 1 retry) because we are
+	// relying on SDK retry mechanism, but we allow 2 attempts to
+	// retry directory listings after XMLSyntaxError
+	pc.SetRetries(2)
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
 		ci:    ci,
 		srv:   objectStorageClient,
 		cache: bucket.NewCache(),
-		pacer: fs.NewPacer(ctx, p),
+		pacer: pc,
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -581,12 +589,7 @@ func (f *Fs) cleanUpBucket(ctx context.Context, bucket string, maxAge time.Durat
 				if operations.SkipDestructive(ctx, what, "remove pending upload") {
 					continue
 				}
-				ignoreErr := f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId)
-				if ignoreErr != nil {
-					// fs.Debugf(f, "ignoring error %s", ignoreErr)
-				}
-			} else {
-				// fs.Debugf(f, "ignoring %s", what)
+				_ = f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId)
 			}
 		} else {
 			fs.Infof(f, "MultipartUpload doesn't have sufficient details to abort.")
@@ -686,6 +689,7 @@ var (
 	_ fs.PutStreamer = &Fs{}
 	_ fs.ListRer     = &Fs{}
 	_ fs.Commander   = &Fs{}
+	_ fs.CleanUpper  = &Fs{}
 
 	_ fs.Object    = &Object{}
 	_ fs.MimeTyper = &Object{}

backend/pikpak/api/types.go

@@ -0,0 +1,535 @@
+// Package api has type definitions for pikpak
+//
+// Manually obtained from the API responses using Browse Dev. Tool and https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"time"
+)
+
+const (
+	// "2022-09-17T14:31:06.056+08:00"
+	timeFormat = `"` + time.RFC3339 + `"`
+)
+
+// Time represents date and time information for the pikpak API, by using RFC3339
+type Time time.Time
+
+// MarshalJSON turns a Time into JSON (in UTC)
+func (t *Time) MarshalJSON() (out []byte, err error) {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return []byte(timeString), nil
+}
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" || string(data) == `""` {
+		return nil
+	}
+	newT, err := time.Parse(timeFormat, string(data))
+	if err != nil {
+		return err
+	}
+	*t = Time(newT)
+	return nil
+}
+
+// Types of things in Item
+const (
+	KindOfFolder        = "drive#folder"
+	KindOfFile          = "drive#file"
+	KindOfFileList      = "drive#fileList"
+	KindOfResumable     = "drive#resumable"
+	KindOfForm          = "drive#form"
+	ThumbnailSizeS      = "SIZE_SMALL"
+	ThumbnailSizeM      = "SIZE_MEDIUM"
+	ThumbnailSizeL      = "SIZE_LARGE"
+	PhaseTypeComplete   = "PHASE_TYPE_COMPLETE"
+	PhaseTypeRunning    = "PHASE_TYPE_RUNNING"
+	PhaseTypeError      = "PHASE_TYPE_ERROR"
+	PhaseTypePending    = "PHASE_TYPE_PENDING"
+	UploadTypeForm      = "UPLOAD_TYPE_FORM"
+	UploadTypeResumable = "UPLOAD_TYPE_RESUMABLE"
+	ListLimit           = 100
+)
+
+// ------------------------------------------------------------
+
+// Error details api error from pikpak
+type Error struct {
+	Reason  string `json:"error"` // short description of the reason, e.g. "file_name_empty" "invalid_request"
+	Code    int    `json:"error_code"`
+	URL     string `json:"error_url,omitempty"`
+	Message string `json:"error_description,omitempty"`
+	// can have either of `error_details` or `details``
+	ErrorDetails []*ErrorDetails `json:"error_details,omitempty"`
+	Details      []*ErrorDetails `json:"details,omitempty"`
+}
+
+// ErrorDetails contains further details of api error
+type ErrorDetails struct {
+	Type     string `json:"@type,omitempty"`
+	Reason   string `json:"reason,omitempty"`
+	Domain   string `json:"domain,omitempty"`
+	Metadata struct {
+	} `json:"metadata,omitempty"` // TODO: undiscovered yet
+	Locale       string        `json:"locale,omitempty"` // e.g. "en"
+	Message      string        `json:"message,omitempty"`
+	StackEntries []interface{} `json:"stack_entries,omitempty"` // TODO: undiscovered yet
+	Detail       string        `json:"detail,omitempty"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e *Error) Error() string {
+	out := fmt.Sprintf("Error %q (%d)", e.Reason, e.Code)
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	return out
+}
+
+// Check Error satisfies the error interface
+var _ error = (*Error)(nil)
+
+// ------------------------------------------------------------
+
+// Filters contains parameters for filters when listing.
+//
+// possible operators
+// * in: a list of comma-separated string
+// * eq: "true" or "false"
+// * gt or lt: time format string, e.g. "2023-01-28T10:56:49.757+08:00"
+type Filters struct {
+	Phase        map[string]string `json:"phase,omitempty"`         // "in" or "eq"
+	Trashed      map[string]bool   `json:"trashed,omitempty"`       // "eq"
+	Kind         map[string]string `json:"kind,omitempty"`          // "eq"
+	Starred      map[string]bool   `json:"starred,omitempty"`       // "eq"
+	ModifiedTime map[string]string `json:"modified_time,omitempty"` // "gt" or "lt"
+}
+
+// Set sets filter values using field name, operator and corresponding value
+func (f *Filters) Set(field, operator, value string) {
+	if value == "" {
+		// UNSET for empty values
+		return
+	}
+	r := reflect.ValueOf(f)
+	fd := reflect.Indirect(r).FieldByName(field)
+	if v, err := strconv.ParseBool(value); err == nil {
+		fd.Set(reflect.ValueOf(map[string]bool{operator: v}))
+	} else {
+		fd.Set(reflect.ValueOf(map[string]string{operator: value}))
+	}
+}
+
+// ------------------------------------------------------------
+// Common Elements
+
+// Link contains a download URL for opening files
+type Link struct {
+	URL    string `json:"url"`
+	Token  string `json:"token"`
+	Expire Time   `json:"expire"`
+	Type   string `json:"type,omitempty"`
+}
+
+// Valid reports whether l is non-nil, has an URL, and is not expired.
+func (l *Link) Valid() bool {
+	return l != nil && l.URL != "" && time.Now().Add(10*time.Second).Before(time.Time(l.Expire))
+}
+
+// URL is a basic form of URL
+type URL struct {
+	Kind string `json:"kind,omitempty"` // e.g. "upload#url"
+	URL  string `json:"url,omitempty"`
+}
+
+// ------------------------------------------------------------
+// Base Elements
+
+// FileList contains a list of File elements
+type FileList struct {
+	Kind            string  `json:"kind,omitempty"` // drive#fileList
+	Files           []*File `json:"files,omitempty"`
+	NextPageToken   string  `json:"next_page_token"`
+	Version         string  `json:"version,omitempty"`
+	VersionOutdated bool    `json:"version_outdated,omitempty"`
+}
+
+// File is a basic element representing a single file object
+//
+// There are two types of download links,
+// 1) one from File.WebContentLink or File.Links.ApplicationOctetStream.URL and
+// 2) the other from File.Medias[].Link.URL.
+// Empirically, 2) is less restrictive to multiple concurrent range-requests
+// for a single file, i.e. supports for higher `--multi-thread-streams=N`.
+// However, it is not generally applicable as it is only for meadia.
+type File struct {
+	Apps              []*FileApp    `json:"apps,omitempty"`
+	Audit             *FileAudit    `json:"audit,omitempty"`
+	Collection        string        `json:"collection,omitempty"` // TODO
+	CreatedTime       Time          `json:"created_time,omitempty"`
+	DeleteTime        Time          `json:"delete_time,omitempty"`
+	FileCategory      string        `json:"file_category,omitempty"`
+	FileExtension     string        `json:"file_extension,omitempty"`
+	FolderType        string        `json:"folder_type,omitempty"`
+	Hash              string        `json:"hash,omitempty"` // sha1 but NOT a valid file hash. looks like a torrent hash
+	IconLink          string        `json:"icon_link,omitempty"`
+	ID                string        `json:"id,omitempty"`
+	Kind              string        `json:"kind,omitempty"` // "drive#file"
+	Links             *FileLinks    `json:"links,omitempty"`
+	Md5Checksum       string        `json:"md5_checksum,omitempty"`
+	Medias            []*Media      `json:"medias,omitempty"`
+	MimeType          string        `json:"mime_type,omitempty"`
+	ModifiedTime      Time          `json:"modified_time,omitempty"` // updated when renamed or moved
+	Name              string        `json:"name,omitempty"`
+	OriginalFileIndex int           `json:"original_file_index,omitempty"` // TODO
+	OriginalURL       string        `json:"original_url,omitempty"`
+	Params            *FileParams   `json:"params,omitempty"`
+	ParentID          string        `json:"parent_id,omitempty"`
+	Phase             string        `json:"phase,omitempty"`
+	Revision          int           `json:"revision,omitempty,string"`
+	Size              int64         `json:"size,omitempty,string"`
+	SortName          string        `json:"sort_name,omitempty"`
+	Space             string        `json:"space,omitempty"`
+	SpellName         []interface{} `json:"spell_name,omitempty"` // TODO maybe list of something?
+	Starred           bool          `json:"starred,omitempty"`
+	ThumbnailLink     string        `json:"thumbnail_link,omitempty"`
+	Trashed           bool          `json:"trashed,omitempty"`
+	UserID            string        `json:"user_id,omitempty"`
+	UserModifiedTime  Time          `json:"user_modified_time,omitempty"`
+	WebContentLink    string        `json:"web_content_link,omitempty"`
+	Writable          bool          `json:"writable,omitempty"`
+}
+
+// FileLinks includes links to file at backend
+type FileLinks struct {
+	ApplicationOctetStream *Link `json:"application/octet-stream,omitempty"`
+}
+
+// FileAudit contains audit information for the file
+type FileAudit struct {
+	Status  string `json:"status,omitempty"` // "STATUS_OK"
+	Message string `json:"message,omitempty"`
+	Title   string `json:"title,omitempty"`
+}
+
+// Media contains info about supported version of media, e.g. original, transcoded, etc
+type Media struct {
+	MediaID   string `json:"media_id,omitempty"`
+	MediaName string `json:"media_name,omitempty"`
+	Video     struct {
+		Height     int    `json:"height,omitempty"`
+		Width      int    `json:"width,omitempty"`
+		Duration   int64  `json:"duration,omitempty"`
+		BitRate    int    `json:"bit_rate,omitempty"`
+		FrameRate  int    `json:"frame_rate,omitempty"`
+		VideoCodec string `json:"video_codec,omitempty"`
+		AudioCodec string `json:"audio_codec,omitempty"`
+		VideoType  string `json:"video_type,omitempty"`
+	} `json:"video,omitempty"`
+	Link           *Link         `json:"link,omitempty"`
+	NeedMoreQuota  bool          `json:"need_more_quota,omitempty"`
+	VipTypes       []interface{} `json:"vip_types,omitempty"` // TODO maybe list of something?
+	RedirectLink   string        `json:"redirect_link,omitempty"`
+	IconLink       string        `json:"icon_link,omitempty"`
+	IsDefault      bool          `json:"is_default,omitempty"`
+	Priority       int           `json:"priority,omitempty"`
+	IsOrigin       bool          `json:"is_origin,omitempty"`
+	ResolutionName string        `json:"resolution_name,omitempty"`
+	IsVisible      bool          `json:"is_visible,omitempty"`
+	Category       string        `json:"category,omitempty"`
+}
+
+// FileParams includes parameters for instant open
+type FileParams struct {
+	Duration     int64  `json:"duration,omitempty,string"` // in seconds
+	Height       int    `json:"height,omitempty,string"`
+	Platform     string `json:"platform,omitempty"` // "Upload"
+	PlatformIcon string `json:"platform_icon,omitempty"`
+	URL          string `json:"url,omitempty"`
+	Width        int    `json:"width,omitempty,string"`
+}
+
+// FileApp includes parameters for instant open
+type FileApp struct {
+	ID            string        `json:"id,omitempty"`   // "decompress" for rar files
+	Name          string        `json:"name,omitempty"` // decompress" for rar files
+	Access        []interface{} `json:"access,omitempty"`
+	Link          string        `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
+	RedirectLink  string        `json:"redirect_link,omitempty"`
+	VipTypes      []interface{} `json:"vip_types,omitempty"`
+	NeedMoreQuota bool          `json:"need_more_quota,omitempty"`
+	IconLink      string        `json:"icon_link,omitempty"`
+	IsDefault     bool          `json:"is_default,omitempty"`
+	Params        struct {
+	} `json:"params,omitempty"` // TODO
+	CategoryIds []interface{} `json:"category_ids,omitempty"`
+	AdSceneType int           `json:"ad_scene_type,omitempty"`
+	Space       string        `json:"space,omitempty"`
+	Links       struct {
+	} `json:"links,omitempty"` // TODO
+}
+
+// ------------------------------------------------------------
+
+// TaskList contains a list of Task elements
+type TaskList struct {
+	Tasks         []*Task `json:"tasks,omitempty"` // "drive#task"
+	NextPageToken string  `json:"next_page_token"`
+	ExpiresIn     int     `json:"expires_in,omitempty"`
+}
+
+// Task is a basic element representing a single task such as offline download and upload
+type Task struct {
+	Kind              string        `json:"kind,omitempty"` // "drive#task"
+	ID                string        `json:"id,omitempty"`   // task id?
+	Name              string        `json:"name,omitempty"` // torrent name?
+	Type              string        `json:"type,omitempty"` // "offline"
+	UserID            string        `json:"user_id,omitempty"`
+	Statuses          []interface{} `json:"statuses,omitempty"`    // TODO
+	StatusSize        int           `json:"status_size,omitempty"` // TODO
+	Params            *TaskParams   `json:"params,omitempty"`      // TODO
+	FileID            string        `json:"file_id,omitempty"`
+	FileName          string        `json:"file_name,omitempty"`
+	FileSize          string        `json:"file_size,omitempty"`
+	Message           string        `json:"message,omitempty"` // e.g. "Saving"
+	CreatedTime       Time          `json:"created_time,omitempty"`
+	UpdatedTime       Time          `json:"updated_time,omitempty"`
+	ThirdTaskID       string        `json:"third_task_id,omitempty"` // TODO
+	Phase             string        `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
+	Progress          int           `json:"progress,omitempty"`
+	IconLink          string        `json:"icon_link,omitempty"`
+	Callback          string        `json:"callback,omitempty"`
+	ReferenceResource interface{}   `json:"reference_resource,omitempty"` // TODO
+	Space             string        `json:"space,omitempty"`
+}
+
+// TaskParams includes parameters informing status of Task
+type TaskParams struct {
+	Age          string `json:"age,omitempty"`
+	PredictSpeed string `json:"predict_speed,omitempty"`
+	PredictType  string `json:"predict_type,omitempty"`
+	URL          string `json:"url,omitempty"`
+}
+
+// Form contains parameters for upload by multipart/form-data
+type Form struct {
+	Headers    struct{} `json:"headers"`
+	Kind       string   `json:"kind"`   // "drive#form"
+	Method     string   `json:"method"` // "POST"
+	MultiParts struct {
+		OSSAccessKeyID string `json:"OSSAccessKeyId"`
+		Signature      string `json:"Signature"`
+		Callback       string `json:"callback"`
+		Key            string `json:"key"`
+		Policy         string `json:"policy"`
+		XUserData      string `json:"x:user_data"`
+	} `json:"multi_parts"`
+	URL string `json:"url"`
+}
+
+// Resumable contains parameters for upload by resumable
+type Resumable struct {
+	Kind     string           `json:"kind,omitempty"`     // "drive#resumable"
+	Provider string           `json:"provider,omitempty"` // e.g. "PROVIDER_ALIYUN"
+	Params   *ResumableParams `json:"params,omitempty"`
+}
+
+// ResumableParams specifies resumable paramegers
+type ResumableParams struct {
+	AccessKeyID     string `json:"access_key_id,omitempty"`
+	AccessKeySecret string `json:"access_key_secret,omitempty"`
+	Bucket          string `json:"bucket,omitempty"`
+	Endpoint        string `json:"endpoint,omitempty"`
+	Expiration      Time   `json:"expiration,omitempty"`
+	Key             string `json:"key,omitempty"`
+	SecurityToken   string `json:"security_token,omitempty"`
+}
+
+// FileInArchive is a basic element in archive
+type FileInArchive struct {
+	Index    int    `json:"index,omitempty"`
+	Filename string `json:"filename,omitempty"`
+	Filesize string `json:"filesize,omitempty"`
+	MimeType string `json:"mime_type,omitempty"`
+	Gcid     string `json:"gcid,omitempty"`
+	Kind     string `json:"kind,omitempty"`
+	IconLink string `json:"icon_link,omitempty"`
+	Path     string `json:"path,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// NewFile is a response to RequestNewFile
+type NewFile struct {
+	File       *File      `json:"file,omitempty"`
+	Form       *Form      `json:"form,omitempty"`
+	Resumable  *Resumable `json:"resumable,omitempty"`
+	Task       *Task      `json:"task,omitempty"`        // null in this case
+	UploadType string     `json:"upload_type,omitempty"` // "UPLOAD_TYPE_FORM" or "UPLOAD_TYPE_RESUMABLE"
+}
+
+// NewTask is a response to RequestNewTask
+type NewTask struct {
+	UploadType string `json:"upload_type,omitempty"` // "UPLOAD_TYPE_URL"
+	File       *File  `json:"file,omitempty"`        // null in this case
+	Task       *Task  `json:"task,omitempty"`
+	URL        *URL   `json:"url,omitempty"` // {"kind": "upload#url"}
+}
+
+// About informs drive status
+type About struct {
+	Kind      string `json:"kind,omitempty"` // "drive#about"
+	Quota     *Quota `json:"quota,omitempty"`
+	ExpiresAt string `json:"expires_at,omitempty"`
+	Quotas    struct {
+	} `json:"quotas,omitempty"` // maybe []*Quota?
+}
+
+// Quota informs drive quota
+type Quota struct {
+	Kind           string `json:"kind,omitempty"`                  // "drive#quota"
+	Limit          int64  `json:"limit,omitempty,string"`          // limit in bytes
+	Usage          int64  `json:"usage,omitempty,string"`          // bytes in use
+	UsageInTrash   int64  `json:"usage_in_trash,omitempty,string"` // bytes in trash but this seems not working
+	PlayTimesLimit string `json:"play_times_limit,omitempty"`      // maybe in seconds
+	PlayTimesUsage string `json:"play_times_usage,omitempty"`      // maybe in seconds
+}
+
+// Share is a response to RequestShare
+//
+// used in PublicLink()
+type Share struct {
+	ShareID   string `json:"share_id,omitempty"`
+	ShareURL  string `json:"share_url,omitempty"`
+	PassCode  string `json:"pass_code,omitempty"`
+	ShareText string `json:"share_text,omitempty"`
+}
+
+// User contains user account information
+//
+// GET https://user.mypikpak.com/v1/user/me
+type User struct {
+	Sub               string          `json:"sub,omitempty"`       // userid for internal use
+	Name              string          `json:"name,omitempty"`      // Username
+	Picture           string          `json:"picture,omitempty"`   // URL to Avatar image
+	Email             string          `json:"email,omitempty"`     // redacted email address
+	Providers         *[]UserProvider `json:"providers,omitempty"` // OAuth provider
+	PhoneNumber       string          `json:"phone_number,omitempty"`
+	Password          string          `json:"password,omitempty"` // "SET" if configured
+	Status            string          `json:"status,omitempty"`   // "ACTIVE"
+	CreatedAt         Time            `json:"created_at,omitempty"`
+	PasswordUpdatedAt Time            `json:"password_updated_at,omitempty"`
+}
+
+// UserProvider details third-party authentication
+type UserProvider struct {
+	ID             string `json:"id,omitempty"` // e.g. "google.com"
+	ProviderUserID string `json:"provider_user_id,omitempty"`
+	Name           string `json:"name,omitempty"` // username
+}
+
+// VIP includes subscription details about premium account
+//
+// GET https://api-drive.mypikpak.com/drive/v1/privilege/vip
+type VIP struct {
+	Result      string `json:"result,omitempty"` // "ACCEPTED"
+	Message     string `json:"message,omitempty"`
+	RedirectURI string `json:"redirect_uri,omitempty"`
+	Data        struct {
+		Expire Time   `json:"expire,omitempty"`
+		Status string `json:"status,omitempty"`  // "invalid" or "ok"
+		Type   string `json:"type,omitempty"`    // "novip" or "platinum"
+		UserID string `json:"user_id,omitempty"` // same as User.Sub
+	} `json:"data,omitempty"`
+}
+
+// DecompressResult is a response to RequestDecompress
+type DecompressResult struct {
+	Status       string `json:"status,omitempty"` // "OK"
+	StatusText   string `json:"status_text,omitempty"`
+	TaskID       string `json:"task_id,omitempty"`   // same as File.Id
+	FilesNum     int    `json:"files_num,omitempty"` // number of files in archive
+	RedirectLink string `json:"redirect_link,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// RequestShare is to request for file share
+type RequestShare struct {
+	FileIds        []string `json:"file_ids,omitempty"`
+	ShareTo        string   `json:"share_to,omitempty"`         // "publiclink",
+	ExpirationDays int      `json:"expiration_days,omitempty"`  // -1 = 'forever'
+	PassCodeOption string   `json:"pass_code_option,omitempty"` // "NOT_REQUIRED"
+}
+
+// RequestBatch is to request for batch actions
+type RequestBatch struct {
+	Ids []string          `json:"ids,omitempty"`
+	To  map[string]string `json:"to,omitempty"`
+}
+
+// RequestNewFile is to request for creating a new `drive#folder` or `drive#file`
+type RequestNewFile struct {
+	// always required
+	Kind       string `json:"kind"` // "drive#folder" or "drive#file"
+	Name       string `json:"name"`
+	ParentID   string `json:"parent_id"`
+	FolderType string `json:"folder_type"`
+	// only when uploading a new file
+	Hash       string            `json:"hash,omitempty"`      // sha1sum
+	Resumable  map[string]string `json:"resumable,omitempty"` // {"provider": "PROVIDER_ALIYUN"}
+	Size       int64             `json:"size,omitempty"`
+	UploadType string            `json:"upload_type,omitempty"` // "UPLOAD_TYPE_FORM" or "UPLOAD_TYPE_RESUMABLE"
+}
+
+// RequestNewTask is to request for creating a new task like offline downloads
+//
+// Name and ParentID can be left empty.
+type RequestNewTask struct {
+	Kind       string `json:"kind,omitempty"` // "drive#file"
+	Name       string `json:"name,omitempty"`
+	ParentID   string `json:"parent_id,omitempty"`
+	UploadType string `json:"upload_type,omitempty"` // "UPLOAD_TYPE_URL"
+	URL        *URL   `json:"url,omitempty"`         // {"url": downloadUrl}
+	FolderType string `json:"folder_type,omitempty"` // "" if parent_id else "DOWNLOAD"
+}
+
+// RequestDecompress is to request for decompress of archive files
+type RequestDecompress struct {
+	Gcid          string           `json:"gcid,omitempty"`     // same as File.Hash
+	Password      string           `json:"password,omitempty"` // ""
+	FileID        string           `json:"file_id,omitempty"`
+	Files         []*FileInArchive `json:"files,omitempty"` // can request selected files to be decompressed
+	DefaultParent bool             `json:"default_parent,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// NOT implemented YET
+
+// RequestArchiveFileList is to request for a list of files in archive
+//
+// POST https://api-drive.mypikpak.com/decompress/v1/list
+type RequestArchiveFileList struct {
+	Gcid     string `json:"gcid,omitempty"`     // same as api.File.Hash
+	Path     string `json:"path,omitempty"`     // "" by default
+	Password string `json:"password,omitempty"` // "" by default
+	FileID   string `json:"file_id,omitempty"`
+}
+
+// ArchiveFileList is a response to RequestArchiveFileList
+type ArchiveFileList struct {
+	Status      string           `json:"status,omitempty"`       // "OK"
+	StatusText  string           `json:"status_text,omitempty"`  // ""
+	TaskID      string           `json:"task_id,omitempty"`      // ""
+	CurrentPath string           `json:"current_path,omitempty"` // ""
+	Title       string           `json:"title,omitempty"`
+	FileSize    int64            `json:"file_size,omitempty"`
+	Gcid        string           `json:"gcid,omitempty"` // same as File.Hash
+	Files       []*FileInArchive `json:"files,omitempty"`
+}

backend/pikpak/helper.go

@@ -0,0 +1,253 @@
+package pikpak
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	"github.com/rclone/rclone/backend/pikpak/api"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Globals
+const (
+	cachePrefix = "rclone-pikpak-sha1sum-"
+)
+
+// requestDecompress requests decompress of compressed files
+func (f *Fs) requestDecompress(ctx context.Context, file *api.File, password string) (info *api.DecompressResult, err error) {
+	req := &api.RequestDecompress{
+		Gcid:          file.Hash,
+		Password:      password,
+		FileID:        file.ID,
+		Files:         []*api.FileInArchive{},
+		DefaultParent: true,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/decompress/v1/decompress",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getUserInfo gets UserInfo from API
+func (f *Fs) getUserInfo(ctx context.Context) (info *api.User, err error) {
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: "https://user.mypikpak.com/v1/user/me",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get userinfo: %w", err)
+	}
+	return
+}
+
+// getVIPInfo gets VIPInfo from API
+func (f *Fs) getVIPInfo(ctx context.Context) (info *api.VIP, err error) {
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: "https://api-drive.mypikpak.com/drive/v1/privilege/vip",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get vip info: %w", err)
+	}
+	return
+}
+
+// requestBatchAction requests batch actions to API
+//
+// action can be one of batch{Copy,Delete,Trash,Untrash}
+func (f *Fs) requestBatchAction(ctx context.Context, action string, req *api.RequestBatch) (err error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/drive/v1/files:" + action,
+		NoResponse: true, // Only returns `{"task_id":""}
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, nil)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("batch action %q failed: %w", action, err)
+	}
+	return nil
+}
+
+// requestNewTask requests a new api.NewTask and returns api.Task
+func (f *Fs) requestNewTask(ctx context.Context, req *api.RequestNewTask) (info *api.Task, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/files",
+	}
+	var newTask api.NewTask
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &newTask)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return newTask.Task, nil
+}
+
+// requestNewFile requests a new api.NewFile and returns api.File
+func (f *Fs) requestNewFile(ctx context.Context, req *api.RequestNewFile) (info *api.NewFile, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/files",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getFile gets api.File from API for the ID passed
+// and returns rich information containing additional fields below
+// * web_content_link
+// * thumbnail_link
+// * links
+// * medias
+func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/drive/v1/files/" + ID,
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		if err == nil && info.Phase != api.PhaseTypeComplete {
+			// could be pending right after file is created/uploaded.
+			return true, errors.New("not PHASE_TYPE_COMPLETE")
+		}
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// patchFile updates attributes of the file by ID
+//
+// currently known patchable fields are
+// * name
+func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api.File, err error) {
+	opts := rest.Opts{
+		Method: "PATCH",
+		Path:   "/drive/v1/files/" + ID,
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getAbout gets drive#quota information from server
+func (f *Fs) getAbout(ctx context.Context) (info *api.About, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/drive/v1/about",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// requestShare returns information about ssharable links
+func (f *Fs) requestShare(ctx context.Context, req *api.RequestShare) (info *api.Share, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/share",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// Read the sha1 of in returning a reader which will read the same contents
+//
+// The cleanup function should be called when out is finished with
+// regardless of whether this function returned an error or not.
+func readSHA1(in io.Reader, size, threshold int64) (sha1sum string, out io.Reader, cleanup func(), err error) {
+	// we need an SHA1
+	hash := sha1.New()
+	// use the teeReader to write to the local file AND calculate the SHA1 while doing so
+	teeReader := io.TeeReader(in, hash)
+
+	// nothing to clean up by default
+	cleanup = func() {}
+
+	// don't cache small files on disk to reduce wear of the disk
+	if size > threshold {
+		var tempFile *os.File
+
+		// create the cache file
+		tempFile, err = os.CreateTemp("", cachePrefix)
+		if err != nil {
+			return
+		}
+
+		_ = os.Remove(tempFile.Name()) // Delete the file - may not work on Windows
+
+		// clean up the file after we are done downloading
+		cleanup = func() {
+			// the file should normally already be close, but just to make sure
+			_ = tempFile.Close()
+			_ = os.Remove(tempFile.Name()) // delete the cache file after we are done - may be deleted already
+		}
+
+		// copy the ENTIRE file to disc and calculate the SHA1 in the process
+		if _, err = io.Copy(tempFile, teeReader); err != nil {
+			return
+		}
+		// jump to the start of the local file so we can pass it along
+		if _, err = tempFile.Seek(0, 0); err != nil {
+			return
+		}
+
+		// replace the already read source with a reader of our cached file
+		out = tempFile
+	} else {
+		// that's a small file, just read it into memory
+		var inData []byte
+		inData, err = io.ReadAll(teeReader)
+		if err != nil {
+			return
+		}
+
+		// set the reader to our read memory block
+		out = bytes.NewReader(inData)
+	}
+	return hex.EncodeToString(hash.Sum(nil)), out, cleanup, nil
+}

backend/pikpak/pikpak_test.go

@@ -0,0 +1,17 @@
+// Test PikPak filesystem interface
+package pikpak_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/pikpak"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestPikPak:",
+		NilObject:  (*pikpak.Object)(nil),
+	})
+}

backend/s3/s3.go

@@ -1,4 +1,4 @@
-// Package s3 provides an interface to Amazon S3 oject storage
+// Package s3 provides an interface to Amazon S3 object storage
 package s3
 
 //go:generate go run gen_setfrom.go -o setfrom.go
@@ -15,7 +15,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -26,6 +25,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
@@ -65,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -104,10 +105,13 @@ func init() {
 				Help:  "Arvan Cloud Object Storage (AOS)",
 			}, {
 				Value: "DigitalOcean",
-				Help:  "Digital Ocean Spaces",
+				Help:  "DigitalOcean Spaces",
 			}, {
 				Value: "Dreamhost",
 				Help:  "Dreamhost DreamObjects",
+			}, {
+				Value: "GCS",
+				Help:  "Google Cloud Storage",
 			}, {
 				Value: "HuaweiOBS",
 				Help:  "Huawei Object Storage Service",
@@ -123,6 +127,9 @@ func init() {
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
+			}, {
+				Value: "Liara",
+				Help:  "Liara Object Storage",
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
@@ -436,7 +443,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -761,6 +768,15 @@ func init() {
 				Value: "s3-eu-south-2.ionoscloud.com",
 				Help:  "Logrono, Spain",
 			}},
+		}, {
+			// Liara endpoints: https://liara.ir/landing/object-storage
+			Name:     "endpoint",
+			Help:     "Endpoint for Liara Object Storage API.",
+			Provider: "Liara",
+			Examples: []fs.OptionExample{{
+				Value: "storage.iran.liara.space",
+				Help:  "The default endpoint\nIran",
+			}},
 		}, {
 			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
 			Name:     "endpoint",
@@ -923,17 +939,19 @@ func init() {
 			}},
 		}, {
 			Name:     "endpoint",
-			Help:     "Endpoint of the Shared Gateway.",
+			Help:     "Endpoint for Google Cloud Storage.",
+			Provider: "GCS",
+			Examples: []fs.OptionExample{{
+				Value: "https://storage.googleapis.com",
+				Help:  "Google Cloud Storage endpoint",
+			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Storj Gateway.",
 			Provider: "Storj",
 			Examples: []fs.OptionExample{{
-				Value: "gateway.eu1.storjshare.io",
-				Help:  "EU1 Shared Gateway",
-			}, {
-				Value: "gateway.us1.storjshare.io",
-				Help:  "US1 Shared Gateway",
-			}, {
-				Value: "gateway.ap1.storjshare.io",
-				Help:  "Asia-Pacific Shared Gateway",
+				Value: "gateway.storjshare.io",
+				Help:  "Global Hosted Gateway",
 			}},
 		}, {
 			// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
@@ -1091,22 +1109,34 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
+			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
+			}, {
+				Value:    "syd1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Sydney 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "sfo3.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces San Francisco 3",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "fra1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Frankfurt 1",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "nyc3.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces New York 3",
+				Help:     "DigitalOcean Spaces New York 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "ams3.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces Amsterdam 3",
+				Help:     "DigitalOcean Spaces Amsterdam 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "sgp1.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces Singapore 1",
+				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
@@ -1126,15 +1156,39 @@ func init() {
 				Provider: "LyveCloud",
 			}, {
 				Value:    "s3.wasabisys.com",
-				Help:     "Wasabi US East endpoint",
+				Help:     "Wasabi US East 1 (N. Virginia)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.us-east-2.wasabisys.com",
+				Help:     "Wasabi US East 2 (N. Virginia)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.us-central-1.wasabisys.com",
+				Help:     "Wasabi US Central 1 (Texas)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.us-west-1.wasabisys.com",
-				Help:     "Wasabi US West endpoint",
+				Help:     "Wasabi US West 1 (Oregon)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ca-central-1.wasabisys.com",
+				Help:     "Wasabi CA Central 1 (Toronto)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.eu-central-1.wasabisys.com",
-				Help:     "Wasabi EU Central endpoint",
+				Help:     "Wasabi EU Central 1 (Amsterdam)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-central-2.wasabisys.com",
+				Help:     "Wasabi EU Central 2 (Frankfurt)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-west-1.wasabisys.com",
+				Help:     "Wasabi EU West 1 (London)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-west-2.wasabisys.com",
+				Help:     "Wasabi EU West 2 (Paris)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.ap-northeast-1.wasabisys.com",
@@ -1144,6 +1198,18 @@ func init() {
 				Value:    "s3.ap-northeast-2.wasabisys.com",
 				Help:     "Wasabi AP Northeast 2 (Osaka) endpoint",
 				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ap-southeast-1.wasabisys.com",
+				Help:     "Wasabi AP Southeast 1 (Singapore)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ap-southeast-2.wasabisys.com",
+				Help:     "Wasabi AP Southeast 2 (Sydney)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "storage.iran.liara.space",
+				Help:     "Liara Iran endpoint",
+				Provider: "Liara",
 			}, {
 				Value:    "s3.ir-thr-at1.arvanstorage.com",
 				Help:     "ArvanCloud Tehran Iran (Asiatech) endpoint",
@@ -1527,7 +1593,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1537,7 +1603,11 @@ This ACL is used for creating objects and if bucket_acl isn't set, for creating
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when server-side copying objects as S3
-doesn't copy the ACL from the source but rather writes a fresh one.`,
+doesn't copy the ACL from the source but rather writes a fresh one.
+
+If the acl is an empty string then no X-Amz-Acl: header is added and
+the default (private) will be used.
+`,
 			Provider: "!Storj,Cloudflare",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
@@ -1591,7 +1661,11 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when only when creating buckets.  If it
-isn't set then "acl" is used instead.`,
+isn't set then "acl" is used instead.
+
+If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl:
+header is added and the default (private) will be used.
+`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "private",
@@ -1752,6 +1826,15 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Value: "STANDARD_IA",
 				Help:  "Infrequent access storage mode",
 			}},
+		}, {
+			// Mapping from here: https://liara.ir/landing/object-storage
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in Liara",
+			Provider: "Liara",
+			Examples: []fs.OptionExample{{
+				Value: "STANDARD",
+				Help:  "Standard storage class",
+			}},
 		}, {
 			// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
 			Name:     "storage_class",
@@ -2164,11 +2247,59 @@ can't check the size and hash but the file contents will be decompressed.
 `,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name: "might_gzip",
+			Help: strings.ReplaceAll(`Set this if the backend might gzip objects.
+
+Normally providers will not alter objects when they are downloaded. If
+an object was not uploaded with |Content-Encoding: gzip| then it won't
+be set on download.
+
+However some providers may gzip objects even if they weren't uploaded
+with |Content-Encoding: gzip| (eg Cloudflare).
+
+A symptom of this would be receiving errors like
+
+    ERROR corrupted on transfer: sizes differ NNN vs MMM
+
+If you set this flag and rclone downloads an object with
+Content-Encoding: gzip set and chunked transfer encoding, then rclone
+will decompress the object on the fly.
+
+If this is set to unset (the default) then rclone will choose
+according to the provider setting what to apply, but you can override
+rclone's choice here.
+`, "|", "`"),
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "use_accept_encoding_gzip",
+			Help: strings.ReplaceAll(`Whether to send |Accept-Encoding: gzip| header.
+
+By default, rclone will append |Accept-Encoding: gzip| to the request to download
+compressed objects whenever possible.
+
+However some providers such as Google Cloud Storage may alter the HTTP headers, breaking
+the signature of the request.
+
+A symptom of this would be receiving errors like
+
+	SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided.
+
+In this case, you might want to try disabling this option.
+`, "|", "`"),
+			Default:  fs.Tristate{},
+			Advanced: true,
 		}, {
 			Name:     "no_system_metadata",
 			Help:     `Suppress setting and reading of system metadata`,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name:     "sts_endpoint",
+			Help:     "Endpoint for STS.\n\nLeave blank if using AWS to use the default endpoint for the region.",
+			Provider: "AWS",
+			Advanced: true,
 		},
 		}})
 }
@@ -2255,6 +2386,7 @@ type Options struct {
 	SecretAccessKey       string               `config:"secret_access_key"`
 	Region                string               `config:"region"`
 	Endpoint              string               `config:"endpoint"`
+	STSEndpoint           string               `config:"sts_endpoint"`
 	LocationConstraint    string               `config:"location_constraint"`
 	ACL                   string               `config:"acl"`
 	BucketACL             string               `config:"bucket_acl"`
@@ -2295,6 +2427,8 @@ type Options struct {
 	Versions              bool                 `config:"versions"`
 	VersionAt             fs.Time              `config:"version_at"`
 	Decompress            bool                 `config:"decompress"`
+	MightGzip             fs.Tristate          `config:"might_gzip"`
+	UseAcceptEncodingGzip fs.Tristate          `config:"use_accept_encoding_gzip"`
 	NoSystemMetadata      bool                 `config:"no_system_metadata"`
 }
 
@@ -2430,7 +2564,7 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -2462,6 +2596,38 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 	}
 }
 
+// Default name resolver
+var defaultResolver = endpoints.DefaultResolver()
+
+// resolve (service, region) to endpoint
+//
+// Used to set endpoint for s3 services and not for other services
+type resolver map[string]string
+
+// Add a service to the resolver, ignoring empty urls
+func (r resolver) addService(service, url string) {
+	if url == "" {
+		return
+	}
+	if !strings.HasPrefix(url, "http") {
+		url = "https://" + url
+	}
+	r[service] = url
+}
+
+// EndpointFor return the endpoint for s3 if set or the default if not
+func (r resolver) EndpointFor(service, region string, opts ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
+	fs.Debugf(nil, "Resolving service %q region %q", service, region)
+	url, ok := r[service]
+	if ok {
+		return endpoints.ResolvedEndpoint{
+			URL:           url,
+			SigningRegion: region,
+		}, nil
+	}
+	return defaultResolver.EndpointFor(service, region, opts...)
+}
+
 // s3Connection makes a connection to s3
 func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
 	ci := fs.GetConfig(ctx)
@@ -2540,8 +2706,12 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
-	if opt.Endpoint != "" {
-		awsConfig.WithEndpoint(opt.Endpoint)
+	if opt.Endpoint != "" || opt.STSEndpoint != "" {
+		// If endpoints are set, override the relevant services only
+		r := make(resolver)
+		r.addService("s3", opt.Endpoint)
+		r.addService("sts", opt.STSEndpoint)
+		awsConfig.WithEndpointResolver(r)
 	}
 
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
@@ -2651,14 +2821,17 @@ func setEndpointValueForIDriveE2(m configmap.Mapper) (err error) {
 // These should be differences from AWS S3
 func setQuirks(opt *Options) {
 	var (
-		listObjectsV2     = true
-		virtualHostStyle  = true
-		urlEncodeListings = true
-		useMultipartEtag  = true
+		listObjectsV2         = true
+		virtualHostStyle      = true
+		urlEncodeListings     = true
+		useMultipartEtag      = true
+		useAcceptEncodingGzip = true
+		mightGzip             = true // assume all providers might gzip until proven otherwise
 	)
 	switch opt.Provider {
 	case "AWS":
 		// No quirks
+		mightGzip = false // Never auto gzips objects
 	case "Alibaba":
 		useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
 	case "HuaweiOBS":
@@ -2695,6 +2868,10 @@ func setQuirks(opt *Options) {
 		// listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Liara":
+		virtualHostStyle = false
+		urlEncodeListings = false
+		useMultipartEtag = false
 	case "LyveCloud":
 		useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS
 	case "Minio":
@@ -2734,6 +2911,10 @@ func setQuirks(opt *Options) {
 	case "Qiniu":
 		useMultipartEtag = false
 		urlEncodeListings = false
+	case "GCS":
+		// Google break request Signature by mutating accept-encoding HTTP header
+		// https://github.com/rclone/rclone/issues/6670
+		useAcceptEncodingGzip = false
 	case "Other":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -2772,6 +2953,18 @@ func setQuirks(opt *Options) {
 		opt.UseMultipartEtag.Valid = true
 		opt.UseMultipartEtag.Value = useMultipartEtag
 	}
+
+	// set MightGzip if not manually set
+	if !opt.MightGzip.Valid {
+		opt.MightGzip.Valid = true
+		opt.MightGzip.Value = mightGzip
+	}
+
+	// set UseAcceptEncodingGzip if not manually set
+	if !opt.UseAcceptEncodingGzip.Valid {
+		opt.UseAcceptEncodingGzip.Valid = true
+		opt.UseAcceptEncodingGzip.Value = useAcceptEncodingGzip
+	}
 }
 
 // setRoot changes the root of the Fs
@@ -2780,6 +2973,14 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
+// return a pointer to the string if non empty or nil if it is empty
+func stringPointerOrNil(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -2797,10 +2998,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, fmt.Errorf("s3: upload cutoff: %w", err)
 	}
 	if opt.Versions && opt.VersionAt.IsSet() {
-		return nil, errors.New("s3: cant use --s3-versions and --s3-version-at at the same time")
-	}
-	if opt.ACL == "" {
-		opt.ACL = "private"
+		return nil, errors.New("s3: can't use --s3-versions and --s3-version-at at the same time")
 	}
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
@@ -2874,6 +3072,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		GetTier:           true,
 		SlowModTime:       true,
 	}).Fill(ctx, f)
+	if opt.Provider == "Storj" {
+		f.features.SetTier = false
+		f.features.GetTier = false
+	}
+	if opt.Provider == "IDrive" {
+		f.features.SetTier = false
+	}
+	// f.listMultipartUploads()
+
 	if f.rootBucket != "" && f.rootDirectory != "" && !opt.NoHeadObject && !strings.HasSuffix(root, "/") {
 		// Check to see if the (bucket,directory) is actually an existing file
 		oldRoot := f.root
@@ -2888,15 +3095,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
-	if opt.Provider == "Storj" {
-		f.features.Copy = nil
-		f.features.SetTier = false
-		f.features.GetTier = false
-	}
-	if opt.Provider == "IDrive" {
-		f.features.SetTier = false
-	}
-	// f.listMultipartUploads()
 	return f, nil
 }
 
@@ -2921,6 +3119,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 	err = f.list(ctx, listOpt{
 		bucket:       bucket,
 		directory:    bucketPath,
+		prefix:       f.rootDirectory,
 		recurse:      true,
 		withVersions: f.opt.Versions,
 		findFile:     true,
@@ -2948,6 +3147,17 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 	return info, versionID, nil
 }
 
+// stringClonePointer clones the string pointed to by sp into new
+// memory. This is useful to stop us keeping references to small
+// strings carved out of large XML responses.
+func stringClonePointer(sp *string) *string {
+	if sp == nil {
+		return nil
+	}
+	var s = *sp
+	return &s
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error ErrorObjectNotFound.
@@ -2973,8 +3183,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		}
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
-		o.storageClass = info.StorageClass
-		o.versionID = versionID
+		o.storageClass = stringClonePointer(info.StorageClass)
+		o.versionID = stringClonePointer(versionID)
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error
 		if err != nil {
@@ -2992,19 +3202,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Gets the bucket location
 func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, error) {
-	req := s3.GetBucketLocationInput{
-		Bucket: &bucket,
-	}
-	var resp *s3.GetBucketLocationOutput
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(ctx, err)
+	region, err := s3manager.GetBucketRegion(ctx, f.ses, bucket, "", func(r *request.Request) {
+		r.Config.S3ForcePathStyle = aws.Bool(f.opt.ForcePathStyle)
 	})
 	if err != nil {
 		return "", err
 	}
-	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
+	return region, nil
 }
 
 // Updates the region for the bucket by reading the region from the
@@ -3118,8 +3322,14 @@ func (f *Fs) newV2List(req *s3.ListObjectsV2Input) bucketLister {
 // Do a V2 listing
 func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) {
 	resp, err = ls.f.c.ListObjectsV2WithContext(ctx, &ls.req)
+	if err != nil {
+		return nil, nil, err
+	}
+	if aws.BoolValue(resp.IsTruncated) && (resp.NextContinuationToken == nil || *resp.NextContinuationToken == "") {
+		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no NextContinuationToken. Should you be using `--s3-list-version 1`?")
+	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
-	return resp, nil, err
+	return resp, nil, nil
 }
 
 // URL Encode the listings
@@ -3300,24 +3510,25 @@ var errEndList = errors.New("end list")
 
 // list options
 type listOpt struct {
-	bucket       string  // bucket to list
-	directory    string  // directory with bucket
-	prefix       string  // prefix to remove from listing
-	addBucket    bool    // if set, the bucket is added to the start of the remote
-	recurse      bool    // if set, recurse to read sub directories
-	withVersions bool    // if set, versions are produced
-	hidden       bool    // if set, return delete markers as objects with size == isDeleteMarker
-	findFile     bool    // if set, it will look for files called (bucket, directory)
-	versionAt    fs.Time // if set only show versions <= this time
+	bucket        string  // bucket to list
+	directory     string  // directory with bucket
+	prefix        string  // prefix to remove from listing
+	addBucket     bool    // if set, the bucket is added to the start of the remote
+	recurse       bool    // if set, recurse to read sub directories
+	withVersions  bool    // if set, versions are produced
+	hidden        bool    // if set, return delete markers as objects with size == isDeleteMarker
+	findFile      bool    // if set, it will look for files called (bucket, directory)
+	versionAt     fs.Time // if set only show versions <= this time
+	noSkipMarkers bool    // if set return dir marker objects
 }
 
 // list lists the objects into the function supplied with the opt
 // supplied.
 func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
+	if opt.prefix != "" {
+		opt.prefix += "/"
+	}
 	if !opt.findFile {
-		if opt.prefix != "" {
-			opt.prefix += "/"
-		}
 		if opt.directory != "" {
 			opt.directory += "/"
 		}
@@ -3421,7 +3632,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 				}
 				remote = remote[len(opt.prefix):]
 				if opt.addBucket {
-					remote = path.Join(opt.bucket, remote)
+					remote = bucket.Join(opt.bucket, remote)
 				}
 				remote = strings.TrimSuffix(remote, "/")
 				err = fn(remote, &s3.Object{Key: &remote}, nil, true)
@@ -3450,10 +3661,10 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 			remote = remote[len(opt.prefix):]
 			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if opt.addBucket {
-				remote = path.Join(opt.bucket, remote)
+				remote = bucket.Join(opt.bucket, remote)
 			}
 			// is this a directory marker?
-			if isDirectory && object.Size != nil && *object.Size == 0 {
+			if isDirectory && object.Size != nil && *object.Size == 0 && !opt.noSkipMarkers {
 				continue // skip directory marker
 			}
 			if versionIDs != nil {
@@ -3676,7 +3887,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 	return f.cache.Create(bucket, func() error {
 		req := s3.CreateBucketInput{
 			Bucket: &bucket,
-			ACL:    &f.opt.BucketACL,
+			ACL:    stringPointerOrNil(f.opt.BucketACL),
 		}
 		if f.opt.LocationConstraint != "" {
 			req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -3741,9 +3952,9 @@ func pathEscape(s string) string {
 // method
 func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) error {
 	req.Bucket = &dstBucket
-	req.ACL = &f.opt.ACL
+	req.ACL = stringPointerOrNil(f.opt.ACL)
 	req.Key = &dstPath
-	source := pathEscape(path.Join(srcBucket, srcPath))
+	source := pathEscape(bucket.Join(srcBucket, srcPath))
 	if src.versionID != nil {
 		source += fmt.Sprintf("?versionId=%s", *src.versionID)
 	}
@@ -3978,9 +4189,9 @@ Usage Examples:
     rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
     rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
 
-This flag also obeys the filters. Test first with -i/--interactive or --dry-run flags
+This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags
 
-    rclone -i backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+    rclone --interactive backend restore --include "*.txt" s3:bucket/path -o priority=Standard
 
 All the objects shown will be marked for restore, then
 
@@ -4048,8 +4259,8 @@ a bucket or with a bucket and path.
 	Long: `This command removes unfinished multipart uploads of age greater than
 max-age which defaults to 24 hours.
 
-Note that you can use -i/--dry-run with this command to see what it
-would do.
+Note that you can use --interactive/-i or --dry-run with this command to see what
+it would do.
 
     rclone backend cleanup s3:bucket/path/to/object
     rclone backend cleanup -o max-age=7w s3:bucket/path/to/object
@@ -4065,8 +4276,8 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
 	Long: `This command removes any old hidden versions of files
 on a versions enabled bucket.
 
-Note that you can use -i/--dry-run with this command to see what it
-would do.
+Note that you can use --interactive/-i or --dry-run with this command to see what
+it would do.
 
     rclone backend cleanup-hidden s3:bucket/path/to/dir
 `,
@@ -4400,13 +4611,14 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 		delErr <- operations.DeleteFiles(ctx, delChan)
 	}()
 	checkErr(f.list(ctx, listOpt{
-		bucket:       bucket,
-		directory:    directory,
-		prefix:       f.rootDirectory,
-		addBucket:    f.rootBucket == "",
-		recurse:      true,
-		withVersions: versioned,
-		hidden:       true,
+		bucket:        bucket,
+		directory:     directory,
+		prefix:        f.rootDirectory,
+		addBucket:     f.rootBucket == "",
+		recurse:       true,
+		withVersions:  versioned,
+		hidden:        true,
+		noSkipMarkers: true,
 	}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
 		if isDirectory {
 			return nil
@@ -4416,7 +4628,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			fs.Errorf(object, "Can't create object %+v", err)
 			return nil
 		}
-		tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
+		tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 		// Work out whether the file is the current version or not
 		isCurrentVersion := !versioned || !version.Match(remote)
 		fs.Debugf(nil, "%q version %v", remote, version.Match(remote))
@@ -4719,23 +4931,12 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 		return nil, err
 	}
 
-	contentLength := &resp.ContentLength
-	if resp.Header.Get("Content-Range") != "" {
-		var contentRange = resp.Header.Get("Content-Range")
-		slash := strings.IndexRune(contentRange, '/')
-		if slash >= 0 {
-			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
-			if err == nil {
-				contentLength = &i
-			} else {
-				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
-			}
-		} else {
-			fs.Debugf(o, "Failed to find length in %q", contentRange)
-		}
+	contentLength := rest.ParseSizeFromHeaders(resp.Header)
+	if contentLength < 0 {
+		fs.Debugf(o, "Failed to parse file size from headers")
 	}
 
-	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
+	lastModified, err := http.ParseTime(resp.Header.Get("Last-Modified"))
 	if err != nil {
 		fs.Debugf(o, "Failed to parse last modified from string %s, %v", resp.Header.Get("Last-Modified"), err)
 	}
@@ -4759,7 +4960,7 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 
 	var head = s3.HeadObjectOutput{
 		ETag:               header("Etag"),
-		ContentLength:      contentLength,
+		ContentLength:      &contentLength,
 		LastModified:       &lastModified,
 		Metadata:           metaData,
 		CacheControl:       header("Cache-Control"),
@@ -4803,7 +5004,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	// Override the automatic decompression in the transport to
 	// download compressed files as-is
-	httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
+	if o.fs.opt.UseAcceptEncodingGzip.Value {
+		httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
+	}
 
 	for _, option := range options {
 		switch option.(type) {
@@ -4858,7 +5061,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	// Decompress body if necessary
 	if aws.StringValue(resp.ContentEncoding) == "gzip" {
-		if o.fs.opt.Decompress {
+		if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
 			return readers.NewGzipReader(resp.Body)
 		}
 		o.fs.warnCompressed.Do(func() {
@@ -4871,7 +5074,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 var warnStreamUpload sync.Once
 
-func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, versionID *string, err error) {
+func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (wantETag, gotETag string, versionID *string, err error) {
 	f := o.fs
 
 	// make concurrency machinery
@@ -4915,11 +5118,13 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return etag, nil, fmt.Errorf("multipart upload failed to initialise: %w", err)
+		return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to initialise: %w", err)
 	}
 	uid := cout.UploadId
 
+	uploadCtx, cancel := context.WithCancel(ctx)
 	defer atexit.OnError(&err, func() {
+		cancel()
 		if o.fs.opt.LeavePartsOnError {
 			return
 		}
@@ -4939,7 +5144,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	})()
 
 	var (
-		g, gCtx  = errgroup.WithContext(ctx)
+		g, gCtx  = errgroup.WithContext(uploadCtx)
 		finished = false
 		partsMu  sync.Mutex // to protect parts
 		parts    []*s3.CompletedPart
@@ -4988,7 +5193,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 			finished = true
 		} else if err != nil {
 			free()
-			return etag, nil, fmt.Errorf("multipart upload failed to read source: %w", err)
+			return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to read source: %w", err)
 		}
 		buf = buf[:n]
 
@@ -5021,7 +5226,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 				uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
 				if err != nil {
 					if partNum <= int64(concurrency) {
-						return f.shouldRetry(ctx, err)
+						return f.shouldRetry(gCtx, err)
 					}
 					// retry all chunks once have done the first batch
 					return true, err
@@ -5043,7 +5248,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	err = g.Wait()
 	if err != nil {
-		return etag, nil, err
+		return wantETag, gotETag, nil, err
 	}
 
 	// sort the completed parts by part number
@@ -5053,7 +5258,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 
 	var resp *s3.CompleteMultipartUploadOutput
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
+		resp, err = f.c.CompleteMultipartUploadWithContext(uploadCtx, &s3.CompleteMultipartUploadInput{
 			Bucket: req.Bucket,
 			Key:    req.Key,
 			MultipartUpload: &s3.CompletedMultipartUpload{
@@ -5062,17 +5267,20 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 			RequestPayer: req.RequestPayer,
 			UploadId:     uid,
 		})
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(uploadCtx, err)
 	})
 	if err != nil {
-		return etag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)
+		return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)
 	}
 	hashOfHashes := md5.Sum(md5s)
-	etag = fmt.Sprintf("%s-%d", hex.EncodeToString(hashOfHashes[:]), len(parts))
+	wantETag = fmt.Sprintf("%s-%d", hex.EncodeToString(hashOfHashes[:]), len(parts))
 	if resp != nil {
+		if resp.ETag != nil {
+			gotETag = *resp.ETag
+		}
 		versionID = resp.VersionId
 	}
-	return etag, versionID, nil
+	return wantETag, gotETag, versionID, nil
 }
 
 // unWrapAwsError unwraps AWS errors, looking for a non AWS error
@@ -5105,7 +5313,7 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec
 		// Can't upload zero length files like this for some reason
 		r.Body = bytes.NewReader([]byte{})
 	} else {
-		r.SetStreamingBody(ioutil.NopCloser(in))
+		r.SetStreamingBody(io.NopCloser(in))
 	}
 	r.SetContext(ctx)
 	r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
@@ -5219,7 +5427,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	req := s3.PutObjectInput{
 		Bucket: &bucket,
-		ACL:    &o.fs.opt.ACL,
+		ACL:    stringPointerOrNil(o.fs.opt.ACL),
 		Key:    &bucketPath,
 	}
 
@@ -5372,52 +5580,58 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	var wantETag string        // Multipart upload Etag to check
-	var gotEtag string         // Etag we got from the upload
+	var gotETag string         // Etag we got from the upload
 	var lastModified time.Time // Time we got from the upload
 	var versionID *string      // versionID we got from the upload
 	if multipart {
-		wantETag, versionID, err = o.uploadMultipart(ctx, &req, size, in)
+		wantETag, gotETag, versionID, err = o.uploadMultipart(ctx, &req, size, in)
 	} else {
 		if o.fs.opt.UsePresignedRequest {
-			gotEtag, lastModified, versionID, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
+			gotETag, lastModified, versionID, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
 		} else {
-			gotEtag, lastModified, versionID, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
+			gotETag, lastModified, versionID, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
 		}
 	}
 	if err != nil {
 		return err
 	}
-	o.versionID = versionID
+	// Only record versionID if we are using --s3-versions or --s3-version-at
+	if o.fs.opt.Versions || o.fs.opt.VersionAt.IsSet() {
+		o.versionID = versionID
+	} else {
+		o.versionID = nil
+	}
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got
 	// uploaded properly. If size < 0 then we need to do the HEAD.
+	var head *s3.HeadObjectOutput
 	if o.fs.opt.NoHead && size >= 0 {
-		var head s3.HeadObjectOutput
-		//structs.SetFrom(&head, &req)
-		setFrom_s3HeadObjectOutput_s3PutObjectInput(&head, &req)
+		head = new(s3.HeadObjectOutput)
+		//structs.SetFrom(head, &req)
+		setFrom_s3HeadObjectOutput_s3PutObjectInput(head, &req)
 		head.ETag = &md5sumHex // doesn't matter quotes are missing
 		head.ContentLength = &size
-		// If we have done a single part PUT request then we can read these
-		if gotEtag != "" {
-			head.ETag = &gotEtag
+		// We get etag back from single and multipart upload so fill it in here
+		if gotETag != "" {
+			head.ETag = &gotETag
 		}
 		if lastModified.IsZero() {
 			lastModified = time.Now()
 		}
 		head.LastModified = &lastModified
 		head.VersionId = versionID
-		o.setMetaData(&head)
-		return nil
-	}
-
-	// Read the metadata from the newly created object
-	o.meta = nil // wipe old metadata
-	head, err := o.headObject(ctx)
-	if err != nil {
-		return err
+	} else {
+		// Read the metadata from the newly created object
+		o.meta = nil // wipe old metadata
+		head, err = o.headObject(ctx)
+		if err != nil {
+			return err
+		}
 	}
 	o.setMetaData(head)
+
+	// Check multipart upload ETag if required
 	if o.fs.opt.UseMultipartEtag.Value && !o.fs.etagIsNotMD5 && wantETag != "" && head.ETag != nil && *head.ETag != "" {
 		gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`)
 		if wantETag != gotETag {

backend/s3/s3_internal_test.go

@@ -6,12 +6,15 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"path"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -250,7 +253,8 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// Create an object
-	const fileName = "test-versions.txt"
+	const dirName = "versions"
+	const fileName = dirName + "/" + "test-versions.txt"
 	contents := random.String(100)
 	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
@@ -280,11 +284,12 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 		}()
 
 		// Read the contents
-		entries, err := f.List(ctx, "")
+		entries, err := f.List(ctx, dirName)
 		require.NoError(t, err)
 		tests := 0
 		var fileNameVersion string
 		for _, entry := range entries {
+			t.Log(entry)
 			remote := entry.Remote()
 			if remote == fileName {
 				t.Run("ReadCurrent", func(t *testing.T) {
@@ -309,6 +314,18 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 			require.NotNil(t, o)
 			assert.Equal(t, int64(100), o.Size(), o.Remote())
 		})
+
+		// Check we can make a NewFs from that object with a version suffix
+		t.Run("NewFs", func(t *testing.T) {
+			newPath := path.Join(fs.ConfigString(f), fileNameVersion)
+			// Make sure --s3-versions is set in the config of the new remote
+			confPath := strings.Replace(newPath, ":", ",versions:", 1)
+			fNew, err := cache.Get(ctx, confPath)
+			// This should return pointing to a file
+			assert.Equal(t, fs.ErrorIsFile, err)
+			// With the directory the directory above
+			assert.Equal(t, dirName, path.Base(fs.ConfigString(fNew)))
+		})
 	})
 
 	t.Run("VersionAt", func(t *testing.T) {

backend/seafile/renew.go

@@ -0,0 +1,54 @@
+package seafile
+
+import (
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Renew allows tokens to be renewed on expiry.
+type Renew struct {
+	ts       *time.Ticker     // timer indicating when it's time to renew the token
+	run      func() error     // the callback to do the renewal
+	done     chan interface{} // channel to end the go routine
+	shutdown *sync.Once
+}
+
+// NewRenew creates a new Renew struct and starts a background process
+// which renews the token whenever it expires.  It uses the run() call
+// to do the renewal.
+func NewRenew(every time.Duration, run func() error) *Renew {
+	r := &Renew{
+		ts:       time.NewTicker(every),
+		run:      run,
+		done:     make(chan interface{}),
+		shutdown: &sync.Once{},
+	}
+	go r.renewOnExpiry()
+	return r
+}
+
+func (r *Renew) renewOnExpiry() {
+	for {
+		select {
+		case <-r.ts.C:
+			err := r.run()
+			if err != nil {
+				fs.Errorf(nil, "error while refreshing decryption token: %s", err)
+			}
+
+		case <-r.done:
+			return
+		}
+	}
+}
+
+// Shutdown stops the ticker and no more renewal will take place.
+func (r *Renew) Shutdown() {
+	// closing a channel can only be done once
+	r.shutdown.Do(func() {
+		r.ts.Stop()
+		close(r.done)
+	})
+}

backend/seafile/renew_test.go

@@ -0,0 +1,34 @@
+package seafile
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShouldAllowShutdownTwice(t *testing.T) {
+	renew := NewRenew(time.Hour, func() error {
+		return nil
+	})
+	renew.Shutdown()
+	renew.Shutdown()
+}
+
+func TestRenewalInTimeLimit(t *testing.T) {
+	var count int64
+
+	renew := NewRenew(100*time.Millisecond, func() error {
+		atomic.AddInt64(&count, 1)
+		return nil
+	})
+	time.Sleep(time.Second)
+	renew.Shutdown()
+
+	// there's no guarantee the CI agent can handle a simple goroutine
+	renewCount := atomic.LoadInt64(&count)
+	t.Logf("renew count = %d", renewCount)
+	assert.Greater(t, renewCount, int64(0))
+	assert.Less(t, renewCount, int64(11))
+}

backend/seafile/seafile.go

@@ -143,6 +143,7 @@ type Fs struct {
 	createDirMutex      sync.Mutex   // Protect creation of directories
 	useOldDirectoryAPI  bool         // Use the old API v2 if seafile < 7
 	moveDirNotAvailable bool         // Version < 7.0 don't have an API to move a directory
+	renew               *Renew       // Renew an encrypted library token
 }
 
 // ------------------------------------------------------------
@@ -268,6 +269,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			}
 			// And remove the public link feature
 			f.features.PublicLink = nil
+
+			// renew the library password every 45 minutes
+			f.renew = NewRenew(45*time.Minute, func() error {
+				return f.authorizeLibrary(context.Background(), libraryID)
+			})
 		}
 	} else {
 		// Deactivate the cleaner feature since there's no library selected
@@ -383,6 +389,15 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 	return nil, fmt.Errorf("unknown state %q", config.State)
 }
 
+// Shutdown the Fs
+func (f *Fs) Shutdown(ctx context.Context) error {
+	if f.renew == nil {
+		return nil
+	}
+	f.renew.Shutdown()
+	return nil
+}
+
 // sets the AuthorizationToken up
 func (f *Fs) setAuthorizationToken(token string) {
 	f.srv.SetHeader("Authorization", "Token "+token)
@@ -1331,6 +1346,7 @@ var (
 	_ fs.PutStreamer  = &Fs{}
 	_ fs.PublicLinker = &Fs{}
 	_ fs.UserInfoer   = &Fs{}
+	_ fs.Shutdowner   = &Fs{}
 	_ fs.Object       = &Object{}
 	_ fs.IDer         = &Object{}
 )

backend/seafile/webapi.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -633,7 +632,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 		})
 		if start > 0 {
 			// We need to read and discard the beginning of the data...
-			_, err = io.CopyN(ioutil.Discard, resp.Body, start)
+			_, err = io.CopyN(io.Discard, resp.Body, start)
 			if err != nil {
 				return nil, err
 			}
@@ -954,11 +953,9 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
 	return nil
 }
 
-// === API v2 from the official documentation, but that have been replaced by the much better v2.1 (undocumented as of Apr 2020)
-// === getDirectoryEntriesAPIv2 is needed to keep compatibility with seafile v6,
-// === the others can probably be removed after the API v2.1 is documented
-
 func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath string) ([]api.DirEntry, error) {
+	// API v2 from the official documentation, but that have been replaced by the much better v2.1 (undocumented as of Apr 2020)
+	// getDirectoryEntriesAPIv2 is needed to keep compatibility with seafile v6.
 	// API Documentation
 	// https://download.seafile.com/published/web-api/v2.1/directories.md#user-content-List%20Items%20in%20Directory
 	if libraryID == "" {
@@ -1002,95 +999,3 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
 	}
 	return result, nil
 }
-
-func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibraryID, dstPath string) (*api.FileInfo, error) {
-	// API Documentation
-	// https://download.seafile.com/published/web-api/v2.1/file.md#user-content-Copy%20File
-	if srcLibraryID == "" || dstLibraryID == "" {
-		return nil, errors.New("libraryID and/or file path argument(s) missing")
-	}
-	srcPath = path.Join("/", srcPath)
-	dstPath = path.Join("/", dstPath)
-
-	// Older API does not seem to accept JSON input here either
-	postParameters := url.Values{
-		"operation": {"copy"},
-		"dst_repo":  {dstLibraryID},
-		"dst_dir":   {f.opt.Enc.FromStandardPath(dstPath)},
-	}
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        APIv20 + srcLibraryID + "/file/",
-		Parameters:  url.Values{"p": {f.opt.Enc.FromStandardPath(srcPath)}},
-		ContentType: "application/x-www-form-urlencoded",
-		Body:        bytes.NewBuffer([]byte(postParameters.Encode())),
-	}
-	result := &api.FileInfo{}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		if resp != nil {
-			if resp.StatusCode == 401 || resp.StatusCode == 403 {
-				return nil, fs.ErrorPermissionDenied
-			}
-		}
-		return nil, fmt.Errorf("failed to copy file %s:'%s' to %s:'%s': %w", srcLibraryID, srcPath, dstLibraryID, dstPath, err)
-	}
-	err = rest.DecodeJSON(resp, &result)
-	if err != nil {
-		return nil, err
-	}
-	return f.decodeFileInfo(result), nil
-}
-
-func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname string) error {
-	// API Documentation
-	// https://download.seafile.com/published/web-api/v2.1/file.md#user-content-Rename%20File
-	if libraryID == "" || newname == "" {
-		return errors.New("libraryID and/or file path argument(s) missing")
-	}
-	filePath = path.Join("/", filePath)
-
-	// No luck with JSON input with the older api2
-	postParameters := url.Values{
-		"operation": {"rename"},
-		"reloaddir": {"true"}, // This is an undocumented trick to avoid an http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
-		"newname":   {f.opt.Enc.FromStandardName(newname)},
-	}
-
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        APIv20 + libraryID + "/file/",
-		Parameters:  url.Values{"p": {f.opt.Enc.FromStandardPath(filePath)}},
-		ContentType: "application/x-www-form-urlencoded",
-		Body:        bytes.NewBuffer([]byte(postParameters.Encode())),
-		NoRedirect:  true,
-		NoResponse:  true,
-	}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		if resp != nil {
-			if resp.StatusCode == 301 {
-				// This is the normal response from the server
-				return nil
-			}
-			if resp.StatusCode == 401 || resp.StatusCode == 403 {
-				return fs.ErrorPermissionDenied
-			}
-			if resp.StatusCode == 404 {
-				return fs.ErrorObjectNotFound
-			}
-		}
-		return fmt.Errorf("failed to rename file: %w", err)
-	}
-	return nil
-}

backend/sftp/sftp.go

@@ -10,7 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	iofs "io/fs"
 	"os"
 	"path"
 	"regexp"
@@ -123,7 +123,10 @@ This enables the use of the following insecure ciphers and key exchange methods:
 - diffie-hellman-group-exchange-sha256
 - diffie-hellman-group-exchange-sha1
 
-Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.`,
+Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.
+
+This must be false if you use either ciphers or key_exchange advanced options.
+`,
 			Default: false,
 			Examples: []fs.OptionExample{
 				{
@@ -321,10 +324,64 @@ Pass multiple variables space separated, eg
 
     VAR1=value VAR2=value
 
-and pass variables with spaces in in quotes, eg
+and pass variables with spaces in quotes, eg
 
     "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
 
+`,
+			Advanced: true,
+		}, {
+			Name:    "ciphers",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of ciphers to be used for session encryption, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q cipher.
+
+This must not be set if use_insecure_cipher is true.
+
+Example:
+
+    aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com
+`,
+			Advanced: true,
+		}, {
+			Name:    "key_exchange",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of key exchange algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q kex.
+
+This must not be set if use_insecure_cipher is true.
+
+Example:
+
+    sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256
+`,
+			Advanced: true,
+		}, {
+			Name:    "macs",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of MACs (message authentication code) algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q mac.
+
+Example:
+
+    umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
+`,
+			Advanced: true,
+		}, {
+			Name:    "host_key_algorithms",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of host key algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q HostKeyAlgorithms.
+
+Note: This can affect the outcome of key negotiation with the server even if server host key validation is not enabled.
+
+Example:
+
+    ssh-ed25519 ssh-rsa ssh-dss
 `,
 			Advanced: true,
 		}},
@@ -362,6 +419,10 @@ type Options struct {
 	ChunkSize               fs.SizeSuffix   `config:"chunk_size"`
 	Concurrency             int             `config:"concurrency"`
 	SetEnv                  fs.SpaceSepList `config:"set_env"`
+	Ciphers                 fs.SpaceSepList `config:"ciphers"`
+	KeyExchange             fs.SpaceSepList `config:"key_exchange"`
+	MACs                    fs.SpaceSepList `config:"macs"`
+	HostKeyAlgorithms       fs.SpaceSepList `config:"host_key_algorithms"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -694,6 +755,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ClientVersion:   "SSH-2.0-" + f.ci.UserAgent,
 	}
 
+	if len(opt.HostKeyAlgorithms) != 0 {
+		sshConfig.HostKeyAlgorithms = []string(opt.HostKeyAlgorithms)
+	}
+
 	if opt.KnownHostsFile != "" {
 		hostcallback, err := knownhosts.New(env.ShellExpand(opt.KnownHostsFile))
 		if err != nil {
@@ -702,10 +767,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		sshConfig.HostKeyCallback = hostcallback
 	}
 
+	if opt.UseInsecureCipher && (opt.Ciphers != nil || opt.KeyExchange != nil) {
+		return nil, fmt.Errorf("use_insecure_cipher must be false if ciphers or key_exchange are set in advanced configuration")
+	}
+
+	sshConfig.Config.SetDefaults()
 	if opt.UseInsecureCipher {
-		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
 		sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
+	} else {
+		if opt.Ciphers != nil {
+			sshConfig.Config.Ciphers = opt.Ciphers
+		}
+		if opt.KeyExchange != nil {
+			sshConfig.Config.KeyExchanges = opt.KeyExchange
+		}
+	}
+
+	if opt.MACs != nil {
+		sshConfig.Config.MACs = opt.MACs
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
@@ -722,10 +802,32 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("couldn't read ssh agent signers: %w", err)
 		}
 		if keyFile != "" {
-			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
+			// If `opt.KeyUseAgent` is false, then it's expected that `opt.KeyFile` contains the private key
+			// and `${opt.KeyFile}.pub` contains the public key.
+			//
+			// If `opt.KeyUseAgent` is true, then it's expected that `opt.KeyFile` contains the public key.
+			// This is how it works with openssh; the `IdentityFile` in openssh config points to the public key.
+			// It's not necessary to specify the public key explicitly when using ssh-agent, since openssh and rclone
+			// will try all the keys they find in the ssh-agent until they find one that works. But just like
+			// `IdentityFile` is used in openssh config to limit the search to one specific key, so does
+			// `opt.KeyFile` in rclone config limit the search to that specific key.
+			//
+			// However, previous versions of rclone would always expect to find the public key in
+			// `${opt.KeyFile}.pub` even if `opt.KeyUseAgent` was true. So for the sake of backward compatibility
+			// we still first attempt to read the public key from `${opt.KeyFile}.pub`. But if it fails with
+			// an `fs.ErrNotExist` then we also try to read the public key from `opt.KeyFile`.
+			pubBytes, err := os.ReadFile(keyFile + ".pub")
 			if err != nil {
-				return nil, fmt.Errorf("failed to read public key file: %w", err)
+				if errors.Is(err, iofs.ErrNotExist) && opt.KeyUseAgent {
+					pubBytes, err = os.ReadFile(keyFile)
+					if err != nil {
+						return nil, fmt.Errorf("failed to read public key file: %w", err)
+					}
+				} else {
+					return nil, fmt.Errorf("failed to read public key file: %w", err)
+				}
 			}
+
 			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse public key file: %w", err)
@@ -747,11 +849,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	}
 
-	// Load key file if specified
-	if keyFile != "" || opt.KeyPem != "" {
+	// Load key file as a private key, if specified. This is only needed when not using an ssh agent.
+	if (keyFile != "" && !opt.KeyUseAgent) || opt.KeyPem != "" {
 		var key []byte
 		if opt.KeyPem == "" {
-			key, err = ioutil.ReadFile(keyFile)
+			key, err = os.ReadFile(keyFile)
 			if err != nil {
 				return nil, fmt.Errorf("failed to read private key file: %w", err)
 			}
@@ -782,7 +884,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 		// If a public key has been specified then use that
 		if pubkeyFile != "" {
-			certfile, err := ioutil.ReadFile(pubkeyFile)
+			certfile, err := os.ReadFile(pubkeyFile)
 			if err != nil {
 				return nil, fmt.Errorf("unable to read cert file: %w", err)
 			}
@@ -915,20 +1017,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 			fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
 			err = session.Run(shellCmd)
 			_ = session.Close()
+			f.shellType = defaultShellType
 			if err != nil {
-				f.shellType = defaultShellType
 				fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
 			} else {
 				outBytes := stdout.Bytes()
 				fs.Debugf(f, "Remote command result: %s", outBytes)
 				outString := string(bytes.TrimSpace(stdout.Bytes()))
-				if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%"
-					f.shellType = "powershell"
-				} else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
-					f.shellType = "cmd"
-				} else { // If Unix: "%ComSpec%"
-					f.shellType = "unix"
-				}
+				if outString != "" {
+					if strings.HasPrefix(outString, "Microsoft.PowerShell") { // PowerShell: "Microsoft.PowerShell%ComSpec%"
+						f.shellType = "powershell"
+					} else if !strings.HasSuffix(outString, "%ComSpec%") { // Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
+						// Additional positive test, to avoid misdetection on unpredicted Unix shell variants
+						s := strings.ToLower(outString)
+						if strings.Contains(s, ".exe") || strings.Contains(s, ".com") {
+							f.shellType = "cmd"
+						}
+					} // POSIX-based Unix shell: "%ComSpec%"
+				} // fish Unix shell: ""
 			}
 		}
 		// Save permanently in config to avoid the extra work next time
@@ -1711,11 +1817,14 @@ func (o *Object) setMetadata(info os.FileInfo) {
 
 // statRemote stats the file or directory at the remote given
 func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
+	absPath := remote
+	if !strings.HasPrefix(remote, "/") {
+		absPath = path.Join(f.absRoot, remote)
+	}
 	c, err := f.getSftpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("stat: %w", err)
 	}
-	absPath := path.Join(f.absRoot, remote)
 	info, err = c.sftpClient.Stat(absPath)
 	f.putSftpConnection(&c, err)
 	return info, err

backend/sharefile/sharefile.go

@@ -77,7 +77,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -479,7 +478,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("failed to open timezone db: %w", err)
 	}
-	tzdata, err := ioutil.ReadAll(timezone)
+	tzdata, err := io.ReadAll(timezone)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read timezone: %w", err)
 	}

backend/sharefile/tzdata_vfsdata.go

@@ -10,7 +10,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	pathpkg "path"
@@ -119,7 +118,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
 	}
 	if f.grPos < f.seekPos {
 		// Fast-forward.
-		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
+		_, err = io.CopyN(io.Discard, f.gr, f.seekPos-f.grPos)
 		if err != nil {
 			return 0, err
 		}

backend/smb/connpool.go

@@ -34,9 +34,10 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{
 		Initiator: &smb2.NTLMInitiator{
-			User:     f.opt.User,
-			Password: pass,
-			Domain:   f.opt.Domain,
+			User:      f.opt.User,
+			Password:  pass,
+			Domain:    f.opt.Domain,
+			TargetSPN: f.opt.SPN,
 		},
 	}
 
@@ -81,7 +82,7 @@ func (c *conn) closed() bool {
 		// list the shares
 		_, nopErr = c.smbSession.ListSharenames()
 	}
-	return nopErr == nil
+	return nopErr != nil
 }
 
 // Show that we are using a SMB session
@@ -103,7 +104,11 @@ func (f *Fs) getSessions() int32 {
 
 // Open a new connection to the SMB server.
 func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) {
-	c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port)
+	// As we are pooling these connections we need to decouple
+	// them from the current context
+	bgCtx := context.Background()
+
+	c, err = f.dial(bgCtx, "tcp", f.opt.Host+":"+f.opt.Port)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SMB: %w", err)
 	}
@@ -114,7 +119,7 @@ func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err erro
 			_ = c.smbSession.Logoff()
 			return nil, fmt.Errorf("couldn't initialize SMB: %w", err)
 		}
-		c.smbShare = c.smbShare.WithContext(ctx)
+		c.smbShare = c.smbShare.WithContext(bgCtx)
 	}
 	return c, nil
 }

backend/smb/smb.go

@@ -60,6 +60,17 @@ func init() {
 			Name:    "domain",
 			Help:    "Domain name for NTLM authentication.",
 			Default: "WORKGROUP",
+		}, {
+			Name: "spn",
+			Help: `Service principal name.
+
+Rclone presents this name to the server. Some servers use this as further
+authentication, and it often needs to be set for clusters. For example:
+
+    cifs/remotehost:1020
+
+Leave blank if not sure.
+`,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -109,6 +120,7 @@ type Options struct {
 	User            string      `config:"user"`
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
+	SPN             string      `config:"spn"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`
@@ -478,11 +490,15 @@ func (f *Fs) makeEntryRelative(share, _path, relative string, stat os.FileInfo)
 }
 
 func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error {
+	dir := path.Dir(_path)
+	if dir == "." {
+		return nil
+	}
 	cn, err := f.getConnection(ctx, share)
 	if err != nil {
 		return err
 	}
-	err = cn.smbShare.MkdirAll(f.toSambaPath(path.Dir(_path)), 0o755)
+	err = cn.smbShare.MkdirAll(f.toSambaPath(dir), 0o755)
 	f.putConnection(&cn)
 	return err
 }

backend/storj/fs.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/text/unicode/norm"
 
 	"storj.io/uplink"
+	"storj.io/uplink/edge"
 )
 
 const (
@@ -31,9 +32,9 @@ const (
 )
 
 var satMap = map[string]string{
-	"us-central-1.storj.io":  "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777",
-	"europe-west-1.storj.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@europe-west-1.tardigrade.io:7777",
-	"asia-east-1.storj.io":   "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@asia-east-1.tardigrade.io:7777",
+	"us1.storj.io": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us1.storj.io:7777",
+	"eu1.storj.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@eu1.storj.io:7777",
+	"ap1.storj.io": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@ap1.storj.io:7777",
 }
 
 // Register with Fs
@@ -105,16 +106,16 @@ func init() {
 				Name:     "satellite_address",
 				Help:     "Satellite address.\n\nCustom satellite address should match the format: `<nodeid>@<address>:<port>`.",
 				Provider: newProvider,
-				Default:  "us-central-1.storj.io",
+				Default:  "us1.storj.io",
 				Examples: []fs.OptionExample{{
-					Value: "us-central-1.storj.io",
-					Help:  "US Central 1",
+					Value: "us1.storj.io",
+					Help:  "US1",
 				}, {
-					Value: "europe-west-1.storj.io",
-					Help:  "Europe West 1",
+					Value: "eu1.storj.io",
+					Help:  "EU1",
 				}, {
-					Value: "asia-east-1.storj.io",
-					Help:  "Asia East 1",
+					Value: "ap1.storj.io",
+					Help:  "AP1",
 				},
 				},
 			},
@@ -156,10 +157,13 @@ type Fs struct {
 
 // Check the interfaces are satisfied.
 var (
-	_ fs.Fs          = &Fs{}
-	_ fs.ListRer     = &Fs{}
-	_ fs.PutStreamer = &Fs{}
-	_ fs.Mover       = &Fs{}
+	_ fs.Fs           = &Fs{}
+	_ fs.ListRer      = &Fs{}
+	_ fs.PutStreamer  = &Fs{}
+	_ fs.Mover        = &Fs{}
+	_ fs.Copier       = &Fs{}
+	_ fs.Purger       = &Fs{}
+	_ fs.PublicLinker = &Fs{}
 )
 
 // NewFs creates a filesystem backed by Storj.
@@ -544,7 +548,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	defer func() {
 		if err != nil {
 			aerr := upload.Abort()
-			if aerr != nil {
+			if aerr != nil && !errors.Is(aerr, uplink.ErrUploadDone) {
 				fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr)
 			}
 		}
@@ -559,6 +563,16 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 	_, err = io.Copy(upload, in)
 	if err != nil {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			// Rclone assumes the backend will create the bucket if not existing yet.
+			// Here we create the bucket and return a retry error for rclone to retry the upload.
+			_, err = f.project.EnsureBucket(ctx, bucketName)
+			if err != nil {
+				return nil, err
+			}
+			return nil, fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried"))
+		}
+
 		err = fserrors.RetryError(err)
 		fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err)
 
@@ -720,3 +734,143 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// Read the new object
 	return f.NewObject(ctx, remote)
 }
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	// Copy parameters
+	srcBucket, srcKey := bucket.Split(srcObj.absolute)
+	dstBucket, dstKey := f.absolute(remote)
+	options := uplink.CopyObjectOptions{}
+
+	// Do the copy
+	newObject, err := f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
+	if err != nil {
+		// Make sure destination bucket exists
+		_, err := f.project.EnsureBucket(ctx, dstBucket)
+		if err != nil {
+			return nil, fmt.Errorf("copy object failed to create destination bucket: %w", err)
+		}
+		// And try again
+		newObject, err = f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
+		if err != nil {
+			return nil, fmt.Errorf("copy object failed: %w", err)
+		}
+	}
+
+	// Return the new object
+	return newObjectFromUplink(f, remote, newObject), nil
+}
+
+// Purge all files in the directory specified
+//
+// Implement this if you have a way of deleting all the files
+// quicker than just running Remove() on the result of List()
+//
+// Return an error if it doesn't exist
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	bucket, directory := f.absolute(dir)
+	if bucket == "" {
+		return errors.New("can't purge from root")
+	}
+
+	if directory == "" {
+		_, err := f.project.DeleteBucketWithObjects(ctx, bucket)
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			return fs.ErrorDirNotFound
+		}
+		return err
+	}
+
+	fs.Infof(directory, "Quick delete is available only for entire bucket. Falling back to list and delete.")
+	objects := f.project.ListObjects(ctx, bucket,
+		&uplink.ListObjectsOptions{
+			Prefix:    directory + "/",
+			Recursive: true,
+		},
+	)
+	if err := objects.Err(); err != nil {
+		return err
+	}
+
+	empty := true
+	for objects.Next() {
+		empty = false
+		_, err := f.project.DeleteObject(ctx, bucket, objects.Item().Key)
+		if err != nil {
+			return err
+		}
+		fs.Infof(objects.Item().Key, "Deleted")
+	}
+
+	if empty {
+		return fs.ErrorDirNotFound
+	}
+
+	return nil
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	bucket, key := f.absolute(remote)
+	if bucket == "" {
+		return "", errors.New("path must be specified")
+	}
+
+	// Rclone requires that a link is only generated if the remote path exists
+	if key == "" {
+		_, err := f.project.StatBucket(ctx, bucket)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		_, err := f.project.StatObject(ctx, bucket, key)
+		if err != nil {
+			if !errors.Is(err, uplink.ErrObjectNotFound) {
+				return "", err
+			}
+			// No object found, check if there is such a prefix
+			iter := f.project.ListObjects(ctx, bucket, &uplink.ListObjectsOptions{Prefix: key + "/"})
+			if iter.Err() != nil {
+				return "", iter.Err()
+			}
+			if !iter.Next() {
+				return "", err
+			}
+		}
+	}
+
+	sharedPrefix := uplink.SharePrefix{Bucket: bucket, Prefix: key}
+
+	permission := uplink.ReadOnlyPermission()
+	if expire.IsSet() {
+		permission.NotAfter = time.Now().Add(time.Duration(expire))
+	}
+
+	sharedAccess, err := f.access.Share(permission, sharedPrefix)
+	if err != nil {
+		return "", fmt.Errorf("sharing access to object failed: %w", err)
+	}
+
+	creds, err := (&edge.Config{
+		AuthServiceAddress: "auth.storjshare.io:7777",
+	}).RegisterAccess(ctx, sharedAccess, &edge.RegisterAccessOptions{Public: true})
+	if err != nil {
+		return "", fmt.Errorf("creating public link failed: %w", err)
+	}
+
+	return edge.JoinShareURL("https://link.storjshare.io", creds.AccessKeyID, bucket, key, nil)
+}

backend/sugarsync/sugarsync_internal_test.go

@@ -2,7 +2,7 @@ package sugarsync
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"testing"
 
@@ -48,7 +48,7 @@ func TestErrorHandler(t *testing.T) {
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			resp := http.Response{
-				Body:       ioutil.NopCloser(bytes.NewBufferString(test.body)),
+				Body:       io.NopCloser(bytes.NewBufferString(test.body)),
 				StatusCode: test.code,
 				Status:     test.status,
 			}

backend/swift/swift.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -1328,23 +1327,6 @@ func (o *Object) removeSegmentsLargeObject(ctx context.Context, containerSegment
 	return nil
 }
 
-func (o *Object) getSegmentsDlo(ctx context.Context) (segmentsContainer string, prefix string, err error) {
-	if err = o.readMetaData(ctx); err != nil {
-		return
-	}
-	dirManifest := o.headers["X-Object-Manifest"]
-	dirManifest, err = url.PathUnescape(dirManifest)
-	if err != nil {
-		return
-	}
-	delimiter := strings.Index(dirManifest, "/")
-	if len(dirManifest) == 0 || delimiter < 0 {
-		err = errors.New("missing or wrong structure of manifest of Dynamic large object")
-		return
-	}
-	return dirManifest[:delimiter], dirManifest[delimiter+1:], nil
-}
-
 // urlEncode encodes a string so that it is a valid URL
 //
 // We don't use any of Go's standard methods as we need `/` not

backend/swift/swift_test.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"io/ioutil"
 	"testing"
 
 	"github.com/ncw/swift/v2"
@@ -136,7 +135,7 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
 	buf := bytes.NewBufferString(contents[:errPosition])
 	errMessage := "potato"
 	er := &readers.ErrorReader{Err: errors.New(errMessage)}
-	in := ioutil.NopCloser(io.MultiReader(buf, er))
+	in := io.NopCloser(io.MultiReader(buf, er))
 
 	file.Size = contentSize
 	obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)

backend/union/entry.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"sync"
 	"time"
 
@@ -87,7 +86,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				errs[i] = fmt.Errorf("%s: %w", o.UpstreamFs().Name(), err)
 				if len(entries) > 1 {
 					// Drain the input buffer to allow other uploads to continue
-					_, _ = io.Copy(ioutil.Discard, readers[i])
+					_, _ = io.Copy(io.Discard, readers[i])
 				}
 			}
 		} else {

backend/union/union.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"path/filepath"
 	"strings"
@@ -501,7 +500,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 			errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
 			if len(upstreams) > 1 {
 				// Drain the input buffer to allow other uploads to continue
-				_, _ = io.Copy(ioutil.Discard, readers[i])
+				_, _ = io.Copy(io.Discard, readers[i])
 			}
 			return
 		}
@@ -757,14 +756,6 @@ func (f *Fs) create(ctx context.Context, path string) ([]*upstream.Fs, error) {
 	return f.createPolicy.Create(ctx, f.upstreams, path)
 }
 
-func (f *Fs) createEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
-	return f.createPolicy.CreateEntries(entries...)
-}
-
-func (f *Fs) search(ctx context.Context, path string) (*upstream.Fs, error) {
-	return f.searchPolicy.Search(ctx, f.upstreams, path)
-}
-
 func (f *Fs) searchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	return f.searchPolicy.SearchEntries(entries...)
 }

backend/uptobox/uptobox.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -215,7 +214,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 
 	client := fshttp.NewClient(ctx)
 	f.srv = rest.NewClient(client).SetRoot(apiBaseURL)
-	f.IDRegexp = regexp.MustCompile("https://uptobox.com/([a-zA-Z0-9]+)")
+	f.IDRegexp = regexp.MustCompile(`^https://uptobox\.com/([a-zA-Z0-9]+)`)
 
 	_, err = f.readMetaDataForPath(ctx, f.dirPath(""), &api.MetadataRequestOptions{Limit: 10})
 	if err != nil {
@@ -239,7 +238,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 func (f *Fs) decodeError(resp *http.Response, response interface{}) (err error) {
 	defer fs.CheckClose(resp.Body, &err)
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}