walk: do filtering in parallel to speed up rclone delete - FIXME okish

See: https://forum.rclone.org/t/rclone-delete-on-s3-very-slow/37543

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -8,9 +8,9 @@ name: build
 on:
   push:
     branches:
-      - '*'
+      - '**'
     tags:
-      - '*'
+      - '**'
   pull_request:
   workflow_dispatch:
     inputs:
@@ -104,7 +104,7 @@ jobs:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
           check-latest: true
@@ -217,7 +217,7 @@ jobs:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
         # Deploy binaries if enabled in config && not a PR && not a fork
-        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
     if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
@@ -237,7 +237,7 @@ jobs:
 
       # Run govulncheck on the latest go version, the one we build binaries with
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.20'
           check-latest: true
@@ -262,7 +262,7 @@ jobs:
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.20'
 
@@ -352,4 +352,4 @@ jobs:
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork
-        if: github.head_ref == '' && github.repository == 'rclone/rclone'
+        if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,61 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v2
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v4
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v2
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Build and publish image
+              uses: docker/build-push-action@v4
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha
+                cache-to: type=gha,mode=max
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252

.github/workflows/build_publish_docker_image.yml

@@ -1,26 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v3
-              with:
-                fetch-depth: 0
-            - name: Build and publish image
-              uses: ilteoood/docker_buildx@1.1.0
-              with:
-                tag: beta
-                imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                publish: true
-                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
-                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/winget.yml

@@ -0,0 +1,14 @@
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: windows-latest # Action can only run on Windows
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.golangci.yml

@@ -2,15 +2,17 @@
 
 linters:
   enable:
-    - deadcode
     - errcheck
     - goimports
     - revive
     - ineffassign
-    - structcheck
-    - varcheck
     - govet
     - unconvert
+    - staticcheck
+    - gosimple
+    - stylecheck
+    - unused
+    - misspell
     #- prealloc
     #- maligned
   disable-all: true
@@ -25,6 +27,30 @@ issues:
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
 
+  exclude-rules:
+
+    - linters:
+      - staticcheck
+      text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
+
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 10m
+
+linters-settings:
+  revive:
+    rules:
+      - name: unreachable-code
+        disabled: true
+      - name: unused-parameter
+        disabled: true
+      - name: empty-block
+        disabled: true
+      - name: redefines-builtin-id
+        disabled: true
+      - name: superfluous-else
+        disabled: true
+  stylecheck:
+    # Only enable the checks performed by the staticcheck stand-alone tool,
+    # as documented here: https://staticcheck.io/docs/configuration/options/#checks
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]

Dockerfile

@@ -11,7 +11,7 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse tzdata && \
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

MAINTAINERS.md

@@ -16,6 +16,7 @@ Current active maintainers of rclone are:
 | Max Sum          | @Max-Sum          | union backend                |
 | Fred             | @creativeprojects | seafile backend              |
 | Caleb Case       | @calebcase        | storj backend                |
+| wiserain         | @wiserain         | pikpak backend               |
 
 **This is a work in progress Draft**
 

README.md

@@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
+  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
@@ -66,6 +67,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
+  * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)

RELEASE.md

@@ -10,7 +10,7 @@ This file describes how to make the various kinds of releases
 ## Making a release
 
   * git checkout master # see below for stable branch
-  * git pull
+  * git pull # IMPORTANT
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
@@ -21,6 +21,7 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
+  * git push origin # without --follow-tags so it doesn't push the tag if it fails
   * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries

VERSION

@@ -1 +1 @@
-v1.62.0
+v1.63.0

backend/all/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
+	_ "github.com/rclone/rclone/backend/pikpak"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"

backend/azureblob/azureblob.go

@@ -690,7 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
-			return nil, fmt.Errorf("create azure enviroment credential failed: %w", err)
+			return nil, fmt.Errorf("create azure environment credential failed: %w", err)
 		}
 	case opt.UseEmulator:
 		if opt.Account == "" {
@@ -953,7 +953,7 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
 }
 
 // Returns whether file is a directory marker or not
-func isDirectoryMarker(size int64, metadata map[string]string, remote string) bool {
+func isDirectoryMarker(size int64, metadata map[string]*string, remote string) bool {
 	// Directory markers are 0 length
 	if size == 0 {
 		endsWithSlash := strings.HasSuffix(remote, "/")
@@ -964,7 +964,7 @@ func isDirectoryMarker(size int64, metadata map[string]string, remote string) bo
 		// defacto standard for marking blobs as directories.
 		// Note also that the metadata hasn't been normalised to lower case yet
 		for k, v := range metadata {
-			if strings.EqualFold(k, "hdi_isfolder") && v == "true" {
+			if v != nil && strings.EqualFold(k, "hdi_isfolder") && *v == "true" {
 				return true
 			}
 		}
@@ -1471,9 +1471,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	srcBlobSVC := srcObj.getBlobSVC()
 	srcURL := srcBlobSVC.URL()
 
-	tier := blob.AccessTier(f.opt.AccessTier)
 	options := blob.StartCopyFromURLOptions{
-		Tier: &tier,
+		Tier: parseTier(f.opt.AccessTier),
 	}
 	var startCopy blob.StartCopyFromURLResponse
 	err = f.pacer.Call(func() (bool, error) {
@@ -1552,12 +1551,15 @@ func (o *Object) Size() int64 {
 	return o.size
 }
 
-func (o *Object) setMetadata(metadata map[string]string) {
+// Set o.metadata from metadata
+func (o *Object) setMetadata(metadata map[string]*string) {
 	if len(metadata) > 0 {
 		// Lower case the metadata
 		o.meta = make(map[string]string, len(metadata))
 		for k, v := range metadata {
-			o.meta[strings.ToLower(k)] = v
+			if v != nil {
+				o.meta[strings.ToLower(k)] = *v
+			}
 		}
 		// Set o.modTime from metadata if it exists and
 		// UseServerModTime isn't in use.
@@ -1573,20 +1575,16 @@ func (o *Object) setMetadata(metadata map[string]string) {
 	}
 }
 
-// Duplicte of setMetadata but taking pointers to strings
-func (o *Object) setMetadataP(metadata map[string]*string) {
-	if len(metadata) > 0 {
-		// Convert the format of the metadata
-		newMeta := make(map[string]string, len(metadata))
-		for k, v := range metadata {
-			if v != nil {
-				newMeta[k] = *v
-			}
-		}
-		o.setMetadata(newMeta)
-	} else {
-		o.meta = nil
+// Get metadata from o.meta
+func (o *Object) getMetadata() (metadata map[string]*string) {
+	if len(o.meta) == 0 {
+		return nil
 	}
+	metadata = make(map[string]*string, len(o.meta))
+	for k, v := range o.meta {
+		metadata[k] = &v
+	}
+	return metadata
 }
 
 // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
@@ -1718,7 +1716,7 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
 	} else {
 		o.accessTier = *info.Properties.AccessTier
 	}
-	o.setMetadataP(metadata)
+	o.setMetadata(metadata)
 
 	return nil
 }
@@ -1729,12 +1727,6 @@ func (o *Object) getBlobSVC() *blob.Client {
 	return o.fs.getBlobSVC(container, directory)
 }
 
-// getBlockBlobSVC creates a block blob client
-func (o *Object) getBlockBlobSVC() *blockblob.Client {
-	container, directory := o.split()
-	return o.fs.getBlockBlobSVC(container, directory)
-}
-
 // clearMetaData clears enough metadata so readMetaData will re-read it
 func (o *Object) clearMetaData() {
 	o.modTime = time.Time{}
@@ -1800,7 +1792,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	blb := o.getBlobSVC()
 	opt := blob.SetMetadataOptions{}
 	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := blb.SetMetadata(ctx, o.meta, &opt)
+		_, err := blb.SetMetadata(ctx, o.getMetadata(), &opt)
 		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1871,48 +1863,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return downloadResponse.Body, nil
 }
 
-// poolWrapper wraps a pool.Pool as an azblob.TransferManager
-type poolWrapper struct {
-	pool     *pool.Pool
-	bufToken chan struct{}
-	runToken chan struct{}
-}
-
-// newPoolWrapper creates an azblob.TransferManager that will use a
-// pool.Pool with maximum concurrency as specified.
-func (f *Fs) newPoolWrapper(concurrency int) *poolWrapper {
-	return &poolWrapper{
-		pool:     f.pool,
-		bufToken: make(chan struct{}, concurrency),
-		runToken: make(chan struct{}, concurrency),
-	}
-}
-
-// Get implements TransferManager.Get().
-func (pw *poolWrapper) Get() []byte {
-	pw.bufToken <- struct{}{}
-	return pw.pool.Get()
-}
-
-// Put implements TransferManager.Put().
-func (pw *poolWrapper) Put(b []byte) {
-	pw.pool.Put(b)
-	<-pw.bufToken
-}
-
-// Run implements TransferManager.Run().
-func (pw *poolWrapper) Run(f func()) {
-	pw.runToken <- struct{}{}
-	go func() {
-		f()
-		<-pw.runToken
-	}()
-}
-
-// Close implements TransferManager.Close().
-func (pw *poolWrapper) Close() {
-}
-
 // Converts a string into a pointer to a string
 func pString(s string) *string {
 	return &s
@@ -2094,10 +2044,9 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		return err
 	}
 
-	tier := blob.AccessTier(o.fs.opt.AccessTier)
 	options := blockblob.CommitBlockListOptions{
-		Metadata:    o.meta,
-		Tier:        &tier,
+		Metadata:    o.getMetadata(),
+		Tier:        parseTier(o.fs.opt.AccessTier),
 		HTTPHeaders: httpHeaders,
 	}
 
@@ -2141,10 +2090,9 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 	b := bytes.NewReader(buf[:n])
 	rs := &readSeekCloser{Reader: b, Seeker: b}
 
-	tier := blob.AccessTier(o.fs.opt.AccessTier)
 	options := blockblob.UploadOptions{
-		Metadata:    o.meta,
-		Tier:        &tier,
+		Metadata:    o.getMetadata(),
+		Tier:        parseTier(o.fs.opt.AccessTier),
 		HTTPHeaders: httpHeaders,
 	}
 
@@ -2314,6 +2262,14 @@ func (o *Object) GetTier() string {
 	return string(o.accessTier)
 }
 
+func parseTier(tier string) *blob.AccessTier {
+	if tier == "" {
+		return nil
+	}
+	msTier := blob.AccessTier(tier)
+	return &msTier
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}

backend/box/box.go

@@ -27,6 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/golang-jwt/jwt/v4"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -45,7 +46,6 @@ import (
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/jws"
 )
 
 const (
@@ -76,6 +76,11 @@ var (
 	}
 )
 
+type boxCustomClaims struct {
+	jwt.RegisteredClaims
+	BoxSubType string `json:"box_sub_type,omitempty"`
+}
+
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -178,7 +183,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 	signingHeaders := getSigningHeaders(boxConfig)
 	queryParams := getQueryParams(boxConfig)
 	client := fshttp.NewClient(ctx)
-	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+	err = jwtutil.Config("box", name, tokenURL, *claims, signingHeaders, queryParams, privateKey, m, client)
 	return err
 }
 
@@ -194,34 +199,29 @@ func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
 	return boxConfig, nil
 }
 
-func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) {
+func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomClaims, err error) {
 	val, err := jwtutil.RandomHex(20)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to generate random string for jti: %w", err)
 	}
 
-	claims = &jws.ClaimSet{
-		Iss: boxConfig.BoxAppSettings.ClientID,
-		Sub: boxConfig.EnterpriseID,
-		Aud: tokenURL,
-		Exp: time.Now().Add(time.Second * 45).Unix(),
-		PrivateClaims: map[string]interface{}{
-			"box_sub_type": boxSubType,
-			"aud":          tokenURL,
-			"jti":          val,
+	claims = &boxCustomClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			ID:        val,
+			Issuer:    boxConfig.BoxAppSettings.ClientID,
+			Subject:   boxConfig.EnterpriseID,
+			Audience:  jwt.ClaimStrings{tokenURL},
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
 		},
+		BoxSubType: boxSubType,
 	}
-
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header {
-	signingHeaders := &jws.Header{
-		Algorithm: "RS256",
-		Typ:       "JWT",
-		KeyID:     boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
+func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
+	signingHeaders := map[string]interface{}{
+		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
-
 	return signingHeaders
 }
 

backend/cache/cache.go

@@ -1787,7 +1787,7 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
 	}
 }
 
-// StopBackgroundRunners will signall all the runners to stop their work
+// StopBackgroundRunners will signal all the runners to stop their work
 // can be triggered from a terminate signal or from testing between runs
 func (f *Fs) StopBackgroundRunners() {
 	f.cleanupChan <- false

backend/cache/cache_internal_test.go

@@ -1098,27 +1098,6 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
-func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = in.Close()
-	}()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = out.Close()
-	}()
-
-	_, err = io.Copy(out, in)
-	return err
-}
-
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 

backend/combine/combine.go

@@ -1,4 +1,4 @@
-// Package combine implents a backend to combine multiple remotes in a directory tree
+// Package combine implements a backend to combine multiple remotes in a directory tree
 package combine
 
 /*
@@ -351,7 +351,7 @@ func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream
 	return g.Wait()
 }
 
-// join the elements together but unline path.Join return empty string
+// join the elements together but unlike path.Join return empty string
 func join(elem ...string) string {
 	result := path.Join(elem...)
 	if result == "." {

backend/crypt/cipher.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/version"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
@@ -178,6 +179,7 @@ type Cipher struct {
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
+	passBadBlocks  bool // if set passed bad blocks as zeroed blocks
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -189,7 +191,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 		dirNameEncrypt: dirNameEncrypt,
 	}
 	c.buffers.New = func() interface{} {
-		return make([]byte, blockSize)
+		return new([blockSize]byte)
 	}
 	err := c.Key(password, salt)
 	if err != nil {
@@ -198,11 +200,16 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
+// Call to set bad block pass through
+func (c *Cipher) setPassBadBlocks(passBadBlocks bool) {
+	c.passBadBlocks = passBadBlocks
+}
+
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
 // If salt is "" we use a fixed salt just to make attackers lives
-// slighty harder than using no salt.
+// slightly harder than using no salt.
 //
 // Note that empty password makes all 0x00 keys which is used in the
 // tests.
@@ -230,15 +237,12 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() []byte {
-	return c.buffers.Get().([]byte)
+func (c *Cipher) getBlock() *[blockSize]byte {
+	return c.buffers.Get().(*[blockSize]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf []byte) {
-	if len(buf) != blockSize {
-		panic("bad blocksize returned to pool")
-	}
+func (c *Cipher) putBlock(buf *[blockSize]byte) {
 	c.buffers.Put(buf)
 }
 
@@ -609,7 +613,7 @@ func (n *nonce) pointer() *[fileNonceSize]byte {
 // fromReader fills the nonce from an io.Reader - normally the OSes
 // crypto random number generator
 func (n *nonce) fromReader(in io.Reader) error {
-	read, err := io.ReadFull(in, (*n)[:])
+	read, err := readers.ReadFill(in, (*n)[:])
 	if read != fileNonceSize {
 		return fmt.Errorf("short read of nonce: %w", err)
 	}
@@ -664,8 +668,8 @@ type encrypter struct {
 	in       io.Reader
 	c        *Cipher
 	nonce    nonce
-	buf      []byte
-	readBuf  []byte
+	buf      *[blockSize]byte
+	readBuf  *[blockSize]byte
 	bufIndex int
 	bufSize  int
 	err      error
@@ -690,9 +694,9 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 		}
 	}
 	// Copy magic into buffer
-	copy(fh.buf, fileMagicBytes)
+	copy((*fh.buf)[:], fileMagicBytes)
 	// Copy nonce into buffer
-	copy(fh.buf[fileMagicSize:], fh.nonce[:])
+	copy((*fh.buf)[fileMagicSize:], fh.nonce[:])
 	return fh, nil
 }
 
@@ -707,22 +711,20 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 	if fh.bufIndex >= fh.bufSize {
 		// Read data
 		// FIXME should overlap the reads with a go-routine and 2 buffers?
-		readBuf := fh.readBuf[:blockDataSize]
-		n, err = io.ReadFull(fh.in, readBuf)
+		readBuf := (*fh.readBuf)[:blockDataSize]
+		n, err = readers.ReadFill(fh.in, readBuf)
 		if n == 0 {
-			// err can't be nil since:
-			// n == len(buf) if and only if err == nil.
 			return fh.finish(err)
 		}
 		// possibly err != nil here, but we will process the
-		// data and the next call to ReadFull will return 0, err
+		// data and the next call to ReadFill will return 0, err
 		// Encrypt the block using the nonce
-		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		secretbox.Seal((*fh.buf)[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufSize])
 	fh.bufIndex += n
 	return n, nil
 }
@@ -763,8 +765,8 @@ type decrypter struct {
 	nonce        nonce
 	initialNonce nonce
 	c            *Cipher
-	buf          []byte
-	readBuf      []byte
+	buf          *[blockSize]byte
+	readBuf      *[blockSize]byte
 	bufIndex     int
 	bufSize      int
 	err          error
@@ -782,9 +784,9 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 		limit:   -1,
 	}
 	// Read file header (magic + nonce)
-	readBuf := fh.readBuf[:fileHeaderSize]
-	_, err := io.ReadFull(fh.rc, readBuf)
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
+	readBuf := (*fh.readBuf)[:fileHeaderSize]
+	n, err := readers.ReadFill(fh.rc, readBuf)
+	if n < fileHeaderSize && err == io.EOF {
 		// This read from 0..fileHeaderSize-1 bytes
 		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
 	} else if err != nil {
@@ -845,10 +847,8 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 func (fh *decrypter) fillBuffer() (err error) {
 	// FIXME should overlap the reads with a go-routine and 2 buffers?
 	readBuf := fh.readBuf
-	n, err := io.ReadFull(fh.rc, readBuf)
+	n, err := readers.ReadFill(fh.rc, (*readBuf)[:])
 	if n == 0 {
-		// err can't be nil since:
-		// n == len(buf) if and only if err == nil.
 		return err
 	}
 	// possibly err != nil here, but we will process the data and
@@ -856,18 +856,25 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	// Check header + 1 byte exists
 	if n <= blockHeaderSize {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	_, ok := secretbox.Open((*fh.buf)[:0], (*readBuf)[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return err // return pending error as it is likely more accurate
 		}
-		return ErrorEncryptedBadBlock
+		if !fh.c.passBadBlocks {
+			return ErrorEncryptedBadBlock
+		}
+		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
+		// Zero out the bad block and continue
+		for i := range (*fh.buf)[:n] {
+			(*fh.buf)[i] = 0
+		}
 	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize
@@ -893,7 +900,7 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	if fh.limit >= 0 && fh.limit < int64(toCopy) {
 		toCopy = int(fh.limit)
 	}
-	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
+	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufIndex+toCopy])
 	fh.bufIndex += n
 	if fh.limit >= 0 {
 		fh.limit -= int64(n)
@@ -904,9 +911,8 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-// calculateUnderlying converts an (offset, limit) in a crypted file
-// into an (underlyingOffset, underlyingLimit) for the underlying
-// file.
+// calculateUnderlying converts an (offset, limit) in an encrypted file
+// into an (underlyingOffset, underlyingLimit) for the underlying file.
 //
 // It also returns number of bytes to discard after reading the first
 // block and number of blocks this is from the start so the nonce can

backend/crypt/cipher_test.go

@@ -27,14 +27,14 @@ func TestNewNameEncryptionMode(t *testing.T) {
 		{"off", NameEncryptionOff, ""},
 		{"standard", NameEncryptionStandard, ""},
 		{"obfuscate", NameEncryptionObfuscated, ""},
-		{"potato", NameEncryptionOff, "Unknown file name encryption mode \"potato\""},
+		{"potato", NameEncryptionOff, "unknown file name encryption mode \"potato\""},
 	} {
 		actual, actualErr := NewNameEncryptionMode(test.in)
 		assert.Equal(t, actual, test.expected)
 		if test.expectedErr == "" {
 			assert.NoError(t, actualErr)
 		} else {
-			assert.Error(t, actualErr, test.expectedErr)
+			assert.EqualError(t, actualErr, test.expectedErr)
 		}
 	}
 }
@@ -726,7 +726,7 @@ func TestNonceFromReader(t *testing.T) {
 	assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
 	buf = bytes.NewBufferString("123456789abcdefghijklmn")
 	err = x.fromReader(buf)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 func TestNonceFromBuf(t *testing.T) {
@@ -1050,7 +1050,7 @@ func TestRandomSource(t *testing.T) {
 	_, _ = source.Read(buf)
 	sink = newRandomSource(1e8)
 	_, err = io.Copy(sink, source)
-	assert.Error(t, err, "Error in stream")
+	assert.EqualError(t, err, "Error in stream at 1")
 }
 
 type zeroes struct{}
@@ -1167,13 +1167,13 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err := c.newEncrypter(z, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
-	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
+	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, (*fh.buf)[:32])
 
 	// Test error path
 	c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "short read of nonce")
+	assert.EqualError(t, err, "short read of nonce: EOF")
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
@@ -1224,7 +1224,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
 		fh, err = c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedFileTooShort.Error())
+		assert.EqualError(t, err, ErrorEncryptedFileTooShort.Error())
 		assert.Equal(t, 1, cd.closed)
 	}
 
@@ -1232,7 +1232,7 @@ func TestNewDecrypter(t *testing.T) {
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
-	assert.Error(t, err, "potato")
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 1, cd.closed)
 
 	// bad magic
@@ -1243,7 +1243,7 @@ func TestNewDecrypter(t *testing.T) {
 		cd := newCloseDetector(bytes.NewBuffer(file0copy))
 		fh, err := c.newDecrypter(cd)
 		assert.Nil(t, fh)
-		assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+		assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 		file0copy[i] ^= 0x1
 		assert.Equal(t, 1, cd.closed)
 	}
@@ -1495,8 +1495,10 @@ func TestDecrypterRead(t *testing.T) {
 		case i == fileHeaderSize:
 			// This would normally produce an error *except* on the first block
 			expectedErr = nil
+		case i <= fileHeaderSize+blockHeaderSize:
+			expectedErr = ErrorEncryptedFileBadHeader
 		default:
-			expectedErr = io.ErrUnexpectedEOF
+			expectedErr = ErrorEncryptedBadBlock
 		}
 		if expectedErr != nil {
 			assert.EqualError(t, err, expectedErr.Error(), what)
@@ -1514,7 +1516,7 @@ func TestDecrypterRead(t *testing.T) {
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
 	_, err = io.ReadAll(fh)
-	assert.Error(t, err, "potato")
+	assert.EqualError(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
 	// Test corrupting the input
@@ -1525,15 +1527,26 @@ func TestDecrypterRead(t *testing.T) {
 		file16copy[i] ^= 0xFF
 		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
-			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
+			assert.EqualError(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
 			_, err = io.ReadAll(fh)
-			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
+			assert.EqualError(t, err, ErrorEncryptedBadBlock.Error())
 		}
 		file16copy[i] ^= 0xFF
 	}
+
+	// Test that we can corrupt a byte and read zeroes if
+	// passBadBlocks is set
+	copy(file16copy, file16)
+	file16copy[len(file16copy)-1] ^= 0xFF
+	c.passBadBlocks = true
+	fh, err = c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
+	assert.NoError(t, err)
+	buf, err := io.ReadAll(fh)
+	assert.NoError(t, err)
+	assert.Equal(t, make([]byte, 16), buf)
 }
 
 func TestDecrypterClose(t *testing.T) {
@@ -1554,7 +1567,7 @@ func TestDecrypterClose(t *testing.T) {
 
 	// double close
 	err = fh.Close()
-	assert.Error(t, err, ErrorFileClosed.Error())
+	assert.EqualError(t, err, ErrorFileClosed.Error())
 	assert.Equal(t, 1, cd.closed)
 
 	// try again reading the file this time
@@ -1581,8 +1594,6 @@ func TestPutGetBlock(t *testing.T) {
 	block := c.getBlock()
 	c.putBlock(block)
 	c.putBlock(block)
-
-	assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) })
 }
 
 func TestKey(t *testing.T) {

backend/crypt/crypt.go

@@ -119,6 +119,15 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
+		}, {
+			Name: "pass_bad_blocks",
+			Help: `If set this will pass bad blocks through as all 0.
+
+This should not be set in normal operation, it should only be set if
+trying to recover a crypted file with errors and it is desired to
+recover as much of the file as possible.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -138,7 +147,7 @@ length and if it's case sensitive.`,
 				},
 				{
 					Value: "base32768",
-					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox)",
 				},
 			},
 			Advanced: true,
@@ -174,6 +183,7 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
+	cipher.setPassBadBlocks(opt.PassBadBlocks)
 	return cipher, nil
 }
 
@@ -262,6 +272,7 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
+	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
 }
 
@@ -454,7 +465,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v crypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}

backend/drive/drive.go

@@ -202,7 +202,7 @@ func init() {
 					m.Set("root_folder_id", "appDataFolder")
 				}
 
-				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
+				if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
 					return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
 						OAuth2Config: driveConfig,
 					})
@@ -598,6 +598,18 @@ resource key is no needed.
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// Don't encode / as it's a valid name character in drive.
 			Default: encoder.EncodeInvalidUtf8,
+		}, {
+			Name:     "env_auth",
+			Help:     "Get IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 
@@ -654,6 +666,7 @@ type Options struct {
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
 	ResourceKey               string               `config:"resource_key"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote drive server
@@ -761,7 +774,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
-			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
+			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
 				fs.Errorf(f, "Received upload limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
@@ -1122,6 +1135,12 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 		if err != nil {
 			return nil, fmt.Errorf("failed to create oauth client from service account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		scopes := driveScopes(opt.Scope)
+		oAuthClient, err = google.DefaultClient(ctx, scopes...)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create client from environment: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
 		if err != nil {
@@ -2880,6 +2899,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			if f.rootFolderID == "appDataFolder" {
 				changesCall.Spaces("appDataFolder")
 			}
+			changesCall.RestrictToMyDrive(!f.opt.SharedWithMe)
 			changeList, err = changesCall.Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})

backend/drive/drive_internal_test.go

@@ -243,6 +243,15 @@ func (f *Fs) InternalTestShouldRetry(t *testing.T) {
 	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
 	assert.False(t, quotaExceededRetry)
 	assert.Equal(t, quotaExceededError, expectedQuotaError)
+
+	sqEItem := googleapi.ErrorItem{
+		Reason: "storageQuotaExceeded",
+	}
+	generic403.Errors[0] = sqEItem
+	expectedStorageQuotaError := fserrors.FatalError(&generic403)
+	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, storageQuotaExceededRetry)
+	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
 }
 
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {

backend/dropbox/batcher.go

@@ -13,7 +13,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -140,49 +139,6 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
 	return complete, nil
 }
 
-// finishBatchJobStatus waits for the batch to complete returning completed entries
-func (b *batcher) finishBatchJobStatus(ctx context.Context, launchBatchStatus *files.UploadSessionFinishBatchLaunch) (complete *files.UploadSessionFinishBatchResult, err error) {
-	if launchBatchStatus.AsyncJobId == "" {
-		return nil, errors.New("wait for batch completion: empty job ID")
-	}
-	var batchStatus *files.UploadSessionFinishBatchJobStatus
-	sleepTime := 100 * time.Millisecond
-	const maxSleepTime = 1 * time.Second
-	startTime := time.Now()
-	try := 1
-	for {
-		remaining := time.Duration(b.f.opt.BatchCommitTimeout) - time.Since(startTime)
-		if remaining < 0 {
-			break
-		}
-		err = b.f.pacer.Call(func() (bool, error) {
-			batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
-				AsyncJobId: launchBatchStatus.AsyncJobId,
-			})
-			return shouldRetry(ctx, err)
-		})
-		if err != nil {
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after error: %v: try %d remaining %v", sleepTime, err, try, remaining)
-		} else {
-			if batchStatus.Tag == "complete" {
-				fs.Debugf(b.f, "Upload batch completed in %v", time.Since(startTime))
-				return batchStatus.Complete, nil
-			}
-			fs.Debugf(b.f, "Wait for batch: sleeping for %v after status: %q: try %d remaining %v", sleepTime, batchStatus.Tag, try, remaining)
-		}
-		time.Sleep(sleepTime)
-		sleepTime *= 2
-		if sleepTime > maxSleepTime {
-			sleepTime = maxSleepTime
-		}
-		try++
-	}
-	if err == nil {
-		err = errors.New("batch didn't complete")
-	}
-	return nil, fmt.Errorf("wait for batch failed after %d tries in %v: %w", try, time.Since(startTime), err)
-}
-
 // commit a batch
 func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []chan<- batcherResponse) (err error) {
 	// If commit fails then signal clients if sync

backend/dropbox/dropbox.go

@@ -536,7 +536,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			default:
 				return nil, err
 			}
-			// if the moint failed we have to abort here
+			// if the mount failed we have to abort here
 		}
 		// if the mount succeeded it's now a normal folder in the users root namespace
 		// we disable shared folder mode and proceed normally

backend/fichier/api.go

@@ -473,7 +473,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("didnt got an upload node: %w", err)
+		return nil, fmt.Errorf("didn't get an upload node: %w", err)
 	}
 
 	// fs.Debugf(f, "Got Upload node")

backend/fichier/fichier.go

@@ -333,7 +333,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
 	if size > int64(300e9) {
-		return nil, errors.New("File too big, cant upload")
+		return nil, errors.New("File too big, can't upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles
 	}

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/ftp"
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -315,18 +315,33 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+// Return a *textproto.Error if err contains one or nil otherwise
+func textprotoError(err error) (errX *textproto.Error) {
+	if errors.As(err, &errX) {
+		return errX
+	}
+	return nil
+}
+
+// returns true if this FTP error should be retried
+func isRetriableFtpError(err error) bool {
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
+			return true
+		}
+	}
+	return false
+}
+
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
+	if isRetriableFtpError(err) {
+		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -463,8 +478,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
-		var tpErr *textproto.Error
-		if !errors.As(err, &tpErr) {
+		if tpErr := textprotoError(err); tpErr != nil {
 			nopErr := c.NoOp()
 			if nopErr != nil {
 				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
@@ -613,8 +627,7 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
@@ -625,8 +638,7 @@ func translateErrorFile(err error) error {
 
 // translateErrorDir turns FTP errors into rclone errors if possible for a directory
 func translateErrorDir(err error) error {
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
@@ -917,8 +929,7 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	}
 	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
@@ -1159,8 +1170,7 @@ func (f *ftpReadCloser) Close() error {
 	// mask the error if it was caused by a premature close
 	// NB StatusAboutToSend is to work around a bug in pureftpd
 	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
-	switch errX := err.(type) {
-	case *textproto.Error:
+	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
@@ -1186,15 +1196,26 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+
+	var (
+		fd *ftp.Response
+		c  *ftp.ServerConn
+	)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		c, err = o.fs.getFtpConnection(ctx)
+		if err != nil {
+			return false, err // getFtpConnection has retries already
+		}
+		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+		if err != nil {
+			o.fs.putFtpConnection(&c, err)
+		}
+		return shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-	if err != nil {
-		o.fs.putFtpConnection(&c, err)
-		return nil, fmt.Errorf("open: %w", err)
-	}
+
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }
@@ -1227,13 +1248,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	// Ignore error 250 here - send by some servers
-	if err != nil {
-		switch errX := err.(type) {
-		case *textproto.Error:
-			switch errX.Code {
-			case ftp.StatusRequestedFileActionOK:
-				err = nil
-			}
+	if errX := textprotoError(err); errX != nil {
+		switch errX.Code {
+		case ftp.StatusRequestedFileActionOK:
+			err = nil
 		}
 	}
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage.go

@@ -82,7 +82,8 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			if saFile != "" || saCreds != "" || anonymous == "true" {
+			envAuth, _ := m.Get("env_auth")
+			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -92,6 +93,9 @@ func init() {
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "project_number",
 			Help: "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+		}, {
+			Name: "user_project",
+			Help: "User project.\n\nOptional - needed only for requester pays.",
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
@@ -330,6 +334,17 @@ can't check the size and hash but the file contents will be decompressed.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:    "env_auth",
+			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default: false,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 }
@@ -337,6 +352,7 @@ can't check the size and hash but the file contents will be decompressed.
 // Options defines the configuration for this backend
 type Options struct {
 	ProjectNumber             string               `config:"project_number"`
+	UserProject               string               `config:"user_project"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
 	Anonymous                 bool                 `config:"anonymous"`
@@ -349,6 +365,7 @@ type Options struct {
 	Decompress                bool                 `config:"decompress"`
 	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote storage server
@@ -500,6 +517,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {
@@ -541,7 +563,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// Check to see if the object exists
 		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
+			get := f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx)
+			if f.opt.UserProject != "" {
+				get = get.UserProject(f.opt.UserProject)
+			}
+			_, err = get.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -601,6 +627,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		directory += "/"
 	}
 	list := f.svc.Objects.List(bucket).Prefix(directory).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		list = list.UserProject(f.opt.UserProject)
+	}
 	if !recurse {
 		list = list.Delimiter("/")
 	}
@@ -707,6 +736,9 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		return nil, errors.New("can't list buckets without project number")
 	}
 	listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks)
+	if f.opt.UserProject != "" {
+		listBuckets = listBuckets.UserProject(f.opt.UserProject)
+	}
 	for {
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
@@ -836,7 +868,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
+			list := f.svc.Objects.List(bucket).MaxResults(1).Context(ctx)
+			if f.opt.UserProject != "" {
+				list = list.UserProject(f.opt.UserProject)
+			}
+			_, err = list.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err == nil {
@@ -871,7 +907,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 			if !f.opt.BucketPolicyOnly {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
-			_, err = insertBucket.Context(ctx).Do()
+			insertBucket = insertBucket.Context(ctx)
+			if f.opt.UserProject != "" {
+				insertBucket = insertBucket.UserProject(f.opt.UserProject)
+			}
+			_, err = insertBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	}, nil)
@@ -896,7 +936,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	}
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
-			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
+			deleteBucket := f.svc.Buckets.Delete(bucket).Context(ctx)
+			if f.opt.UserProject != "" {
+				deleteBucket = deleteBucket.UserProject(f.opt.UserProject)
+			}
+			err = deleteBucket.Do()
 			return shouldRetry(ctx, err)
 		})
 	})
@@ -942,7 +986,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var rewriteResponse *storage.RewriteResponse
 	for {
 		err = f.pacer.Call(func() (bool, error) {
-			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
+			rewriteRequest = rewriteRequest.Context(ctx)
+			if f.opt.UserProject != "" {
+				rewriteRequest.UserProject(f.opt.UserProject)
+			}
+			rewriteResponse, err = rewriteRequest.Do()
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1053,7 +1101,11 @@ func (o *Object) setMetaData(info *storage.Object) {
 func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
+		get := o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			get = get.UserProject(o.fs.opt.UserProject)
+		}
+		object, err = get.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1125,7 +1177,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
 		if !o.fs.opt.BucketPolicyOnly {
 			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = copyObject.Context(ctx).Do()
+		copyObject = copyObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			copyObject = copyObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = copyObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1142,6 +1198,9 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.fs.opt.UserProject != "" {
+		o.url = o.url + "&userProject=" + o.fs.opt.UserProject
+	}
 	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
 	if err != nil {
 		return nil, err
@@ -1234,7 +1293,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if !o.fs.opt.BucketPolicyOnly {
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = insertObject.Context(ctx).Do()
+		insertObject = insertObject.Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			insertObject = insertObject.UserProject(o.fs.opt.UserProject)
+		}
+		newObject, err = insertObject.Do()
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1249,7 +1312,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
+		deleteBucket := o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx)
+		if o.fs.opt.UserProject != "" {
+			deleteBucket = deleteBucket.UserProject(o.fs.opt.UserProject)
+		}
+		err = deleteBucket.Do()
 		return shouldRetry(ctx, err)
 	})
 	return err

backend/hdfs/hdfs.go

@@ -42,9 +42,9 @@ for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
 			Help: `Kerberos data transfer protection: authentication|integrity|privacy.
 
 Specifies whether or not authentication, data signature integrity
-checks, and wire encryption is required when communicating the the
-datanodes. Possible values are 'authentication', 'integrity' and
-'privacy'. Used only with KERBEROS enabled.`,
+checks, and wire encryption are required when communicating with
+the datanodes. Possible values are 'authentication', 'integrity'
+and 'privacy'. Used only with KERBEROS enabled.`,
 			Examples: []fs.OptionExample{{
 				Value: "privacy",
 				Help:  "Ensure authentication, integrity and encryption enabled.",

backend/hidrive/helpers.go

@@ -294,15 +294,6 @@ func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType Cop
 	return &result, nil
 }
 
-// copyDirectory moves the directory at the source-path to the destination-path and
-// returns the resulting api-object if successful.
-//
-// The operation will only be successful
-// if the parent-directory of the destination-path exists.
-func (f *Fs) copyDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
-	return f.copyOrMove(ctx, true, CopyOriginalPreserveModTime, source, destination, onExist)
-}
-
 // moveDirectory moves the directory at the source-path to the destination-path and
 // returns the resulting api-object if successful.
 //

backend/hidrive/hidrive.go

@@ -2,7 +2,7 @@
 package hidrive
 
 // FIXME HiDrive only supports file or folder names of 255 characters or less.
-// Operations that create files oder folder with longer names will throw a HTTP error:
+// Operations that create files or folders with longer names will throw an HTTP error:
 // - 422 Unprocessable Entity
 // A more graceful way for rclone to handle this may be desirable.
 
@@ -338,7 +338,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("could not access root-prefix: %w", err)
 		}
 		if item.Type != api.HiDriveObjectTypeDirectory {
-			return nil, errors.New("The root-prefix needs to point to a valid directory or be empty")
+			return nil, errors.New("the root-prefix needs to point to a valid directory or be empty")
 		}
 	}
 

backend/jottacloud/jottacloud.go

@@ -1838,12 +1838,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err == nil {
 			// if the object exists delete it
 			err = o.remove(ctx, true)
-			if err != nil {
+			if err != nil && err != fs.ErrorObjectNotFound {
+				// if delete failed then report that, unless it was because the file did not exist after all
 				return fmt.Errorf("failed to remove old object: %w", err)
 			}
-		}
-		// if the object does not exist we can just continue but if the error is something different we should report that
-		if err != fs.ErrorObjectNotFound {
+		} else if err != fs.ErrorObjectNotFound {
+			// if the object does not exist we can just continue but if the error is something different we should report that
 			return err
 		}
 	}
@@ -1930,7 +1930,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.md5 = result.Md5
 		o.modTime = time.Unix(result.Modified/1000, 0)
 	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
+		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
 		return o.readMetaData(ctx, true)
 	}
 
@@ -1951,10 +1951,17 @@ func (o *Object) remove(ctx context.Context, hard bool) error {
 		opts.Parameters.Set("dl", "true")
 	}
 
-	return o.fs.pacer.Call(func() (bool, error) {
+	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
 		return shouldRetry(ctx, resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// attempting to hard delete will fail if path does not exist, but standard delete will succeed
+		if apiErr.StatusCode == http.StatusNotFound {
+			return fs.ErrorObjectNotFound
+		}
+	}
+	return err
 }
 
 // Remove an object

backend/local/local.go

@@ -266,7 +266,10 @@ type Object struct {
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+var (
+	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
+)
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -310,7 +313,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
+	// Check to see if this is a .rclonelink if not found
+	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
+	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
+		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
+	}
 	if err == nil && f.isRegular(fi.Mode()) {
+		// Handle the odd case, that a symlink was specified by name without the link suffix
+		if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+			return nil, errLinksNeedsSuffix
+		}
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -524,6 +536,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
+				// Quietly skip errors on excluded files and directories
+				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
@@ -536,11 +552,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				}
 				mode = fi.Mode()
 			}
-			// Don't include non directory if not included
-			// we leave directory filtering to the layer above
-			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
-				continue
-			}
 			if fi.IsDir() {
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
@@ -553,6 +564,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
+				// Don't include non directory if not included
+				// we leave directory filtering to the layer above
+				if useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err

backend/local/local_internal_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
@@ -145,6 +146,20 @@ func TestSymlink(t *testing.T) {
 	_, err = r.Flocal.NewObject(ctx, "symlink2.txt")
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
+	// Check that NewFs works with the suffixed version and --links
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.Equal(t, dir, f2.(*Fs).root)
+
+	// Check that NewFs doesn't see the non suffixed version with --links
+	f2, err = NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"), configmap.Simple{
+		"links": "true",
+	})
+	require.Equal(t, errLinksNeedsSuffix, err)
+	require.Nil(t, f2)
+
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
@@ -395,3 +410,107 @@ func TestFilter(t *testing.T) {
 	sort.Sort(entries)
 	require.Equal(t, "[included]", fmt.Sprint(entries))
 }
+
+func testFilterSymlink(t *testing.T, copyLinks bool) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
+	r.WriteFile("included.file", "included file", when)
+	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
+	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
+	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
+	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
+
+	defer func() {
+		// Reset -L/-l mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Lstat
+	}()
+	if copyLinks {
+		// Set fs into "-L" mode
+		f.opt.FollowSymlinks = true
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Stat
+	} else {
+		// Set fs into "-l" mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = true
+		f.lstat = os.Lstat
+	}
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Reset global error count
+	accounting.Stats(ctx).ResetErrors()
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included.file"))
+	require.NoError(t, fi.AddRule("+ included.dir/**"))
+	if copyLinks {
+		require.NoError(t, fi.AddRule("+ included.file.link"))
+		require.NoError(t, fi.AddRule("+ included.dir.link/**"))
+	} else {
+		require.NoError(t, fi.AddRule("+ included.file.link.rclonelink"))
+		require.NoError(t, fi.AddRule("+ included.dir.link.rclonelink"))
+	}
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+
+	if copyLinks {
+		// Check 1 global errors one for each dangling symlink
+		assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+	} else {
+		// Check 0 global errors as dangling symlink copied properly
+		assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+	}
+	accounting.Stats(ctx).ResetErrors()
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[dangling.link.rclonelink included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	if copyLinks {
+		require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+	} else {
+		require.Equal(t, "[included.dir included.dir.link.rclonelink included.file included.file.link.rclonelink]", fmt.Sprint(entries))
+	}
+
+	// Check listing through a symlink still works
+	entries, err = f.List(ctx, "included.dir")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
+}
+
+func TestFilterSymlinkCopyLinks(t *testing.T) {
+	testFilterSymlink(t, true)
+}
+
+func TestFilterSymlinkLinks(t *testing.T) {
+	testFilterSymlink(t, false)
+}

backend/local/metadata_linux.go

@@ -91,7 +91,7 @@ func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
 		// The types of t.Sec and t.Nsec vary from int32 to int64 on
 		// different Linux architectures so we need to cast them to
 		// int64 here and hence need to quiet the linter about
-		// unecessary casts.
+		// unnecessary casts.
 		//
 		// nolint: unconvert
 		m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat))

backend/mega/mega.go

@@ -90,7 +90,7 @@ permanently delete objects instead.`,
 MEGA uses plain text HTTP connections by default.
 Some ISPs throttle HTTP connections, this causes transfers to become very slow.
 Enabling this will force MEGA to use HTTPS for all transfers.
-HTTPS is normally not necesary since all data is already encrypted anyway.
+HTTPS is normally not necessary since all data is already encrypted anyway.
 Enabling it will increase CPU usage and add network overhead.`,
 			Default:  false,
 			Advanced: true,

backend/netstorage/netstorage.go

@@ -819,6 +819,8 @@ func (f *Fs) getAuth(req *http.Request) error {
 	// Set Authorization header
 	dataHeader := generateDataHeader(f)
 	path := req.URL.RequestURI()
+	//lint:ignore SA1008 false positive when running staticcheck, the header name is according to docs even if not canonical
+	//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1008
 	actionHeader := req.Header["X-Akamai-ACS-Action"][0]
 	fs.Debugf(nil, "NetStorage API %s call %s for path %q", req.Method, actionHeader, path)
 	req.Header.Set("X-Akamai-ACS-Auth-Data", dataHeader)

backend/onedrive/onedrive.go

@@ -265,7 +265,7 @@ At the time of writing this only works with OneDrive personal paid accounts.
 			Help: `Specify the hash in use for the backend.
 
 This specifies the hash type in use. If set to "auto" it will use the
-default hash which is is QuickXorHash.
+default hash which is QuickXorHash.
 
 Before rclone 1.62 an SHA1 hash was used by default for Onedrive
 Personal. For 1.62 and later the default is to use a QuickXorHash for
@@ -1724,6 +1724,10 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	token := make(chan struct{}, f.ci.Checkers)
 	var wg sync.WaitGroup
 	err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+		if err != nil {
+			fs.Errorf(f, "Failed to list %q: %v", path, err)
+			return nil
+		}
 		err = entries.ForObjectError(func(obj fs.Object) error {
 			o, ok := obj.(*Object)
 			if !ok {

backend/onedrive/quickxorhash/quickxorhash.go

@@ -61,7 +61,7 @@ func New() hash.Hash {
 func (q *quickXorHash) Write(p []byte) (n int, err error) {
 	var i int
 	// fill last remain
-	lastRemain := int(q.size) % dataSize
+	lastRemain := q.size % dataSize
 	if lastRemain != 0 {
 		i += xorBytes(q.data[lastRemain:], p)
 	}

backend/oracleobjectstorage/options.go

@@ -289,7 +289,7 @@ Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/using
 		}},
 	}, {
 		Name: "sse_kms_key_id",
-		Help: `if using using your own master key in vault, this header specifies the 
+		Help: `if using your own master key in vault, this header specifies the
 OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
 the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
 Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`,

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -589,12 +589,7 @@ func (f *Fs) cleanUpBucket(ctx context.Context, bucket string, maxAge time.Durat
 				if operations.SkipDestructive(ctx, what, "remove pending upload") {
 					continue
 				}
-				ignoreErr := f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId)
-				if ignoreErr != nil {
-					// fs.Debugf(f, "ignoring error %s", ignoreErr)
-				}
-			} else {
-				// fs.Debugf(f, "ignoring %s", what)
+				_ = f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId)
 			}
 		} else {
 			fs.Infof(f, "MultipartUpload doesn't have sufficient details to abort.")

backend/pikpak/api/types.go

@@ -0,0 +1,535 @@
+// Package api has type definitions for pikpak
+//
+// Manually obtained from the API responses using Browse Dev. Tool and https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"time"
+)
+
+const (
+	// "2022-09-17T14:31:06.056+08:00"
+	timeFormat = `"` + time.RFC3339 + `"`
+)
+
+// Time represents date and time information for the pikpak API, by using RFC3339
+type Time time.Time
+
+// MarshalJSON turns a Time into JSON (in UTC)
+func (t *Time) MarshalJSON() (out []byte, err error) {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return []byte(timeString), nil
+}
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" || string(data) == `""` {
+		return nil
+	}
+	newT, err := time.Parse(timeFormat, string(data))
+	if err != nil {
+		return err
+	}
+	*t = Time(newT)
+	return nil
+}
+
+// Types of things in Item
+const (
+	KindOfFolder        = "drive#folder"
+	KindOfFile          = "drive#file"
+	KindOfFileList      = "drive#fileList"
+	KindOfResumable     = "drive#resumable"
+	KindOfForm          = "drive#form"
+	ThumbnailSizeS      = "SIZE_SMALL"
+	ThumbnailSizeM      = "SIZE_MEDIUM"
+	ThumbnailSizeL      = "SIZE_LARGE"
+	PhaseTypeComplete   = "PHASE_TYPE_COMPLETE"
+	PhaseTypeRunning    = "PHASE_TYPE_RUNNING"
+	PhaseTypeError      = "PHASE_TYPE_ERROR"
+	PhaseTypePending    = "PHASE_TYPE_PENDING"
+	UploadTypeForm      = "UPLOAD_TYPE_FORM"
+	UploadTypeResumable = "UPLOAD_TYPE_RESUMABLE"
+	ListLimit           = 100
+)
+
+// ------------------------------------------------------------
+
+// Error details api error from pikpak
+type Error struct {
+	Reason  string `json:"error"` // short description of the reason, e.g. "file_name_empty" "invalid_request"
+	Code    int    `json:"error_code"`
+	URL     string `json:"error_url,omitempty"`
+	Message string `json:"error_description,omitempty"`
+	// can have either of `error_details` or `details``
+	ErrorDetails []*ErrorDetails `json:"error_details,omitempty"`
+	Details      []*ErrorDetails `json:"details,omitempty"`
+}
+
+// ErrorDetails contains further details of api error
+type ErrorDetails struct {
+	Type     string `json:"@type,omitempty"`
+	Reason   string `json:"reason,omitempty"`
+	Domain   string `json:"domain,omitempty"`
+	Metadata struct {
+	} `json:"metadata,omitempty"` // TODO: undiscovered yet
+	Locale       string        `json:"locale,omitempty"` // e.g. "en"
+	Message      string        `json:"message,omitempty"`
+	StackEntries []interface{} `json:"stack_entries,omitempty"` // TODO: undiscovered yet
+	Detail       string        `json:"detail,omitempty"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e *Error) Error() string {
+	out := fmt.Sprintf("Error %q (%d)", e.Reason, e.Code)
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	return out
+}
+
+// Check Error satisfies the error interface
+var _ error = (*Error)(nil)
+
+// ------------------------------------------------------------
+
+// Filters contains parameters for filters when listing.
+//
+// possible operators
+// * in: a list of comma-separated string
+// * eq: "true" or "false"
+// * gt or lt: time format string, e.g. "2023-01-28T10:56:49.757+08:00"
+type Filters struct {
+	Phase        map[string]string `json:"phase,omitempty"`         // "in" or "eq"
+	Trashed      map[string]bool   `json:"trashed,omitempty"`       // "eq"
+	Kind         map[string]string `json:"kind,omitempty"`          // "eq"
+	Starred      map[string]bool   `json:"starred,omitempty"`       // "eq"
+	ModifiedTime map[string]string `json:"modified_time,omitempty"` // "gt" or "lt"
+}
+
+// Set sets filter values using field name, operator and corresponding value
+func (f *Filters) Set(field, operator, value string) {
+	if value == "" {
+		// UNSET for empty values
+		return
+	}
+	r := reflect.ValueOf(f)
+	fd := reflect.Indirect(r).FieldByName(field)
+	if v, err := strconv.ParseBool(value); err == nil {
+		fd.Set(reflect.ValueOf(map[string]bool{operator: v}))
+	} else {
+		fd.Set(reflect.ValueOf(map[string]string{operator: value}))
+	}
+}
+
+// ------------------------------------------------------------
+// Common Elements
+
+// Link contains a download URL for opening files
+type Link struct {
+	URL    string `json:"url"`
+	Token  string `json:"token"`
+	Expire Time   `json:"expire"`
+	Type   string `json:"type,omitempty"`
+}
+
+// Valid reports whether l is non-nil, has an URL, and is not expired.
+func (l *Link) Valid() bool {
+	return l != nil && l.URL != "" && time.Now().Add(10*time.Second).Before(time.Time(l.Expire))
+}
+
+// URL is a basic form of URL
+type URL struct {
+	Kind string `json:"kind,omitempty"` // e.g. "upload#url"
+	URL  string `json:"url,omitempty"`
+}
+
+// ------------------------------------------------------------
+// Base Elements
+
+// FileList contains a list of File elements
+type FileList struct {
+	Kind            string  `json:"kind,omitempty"` // drive#fileList
+	Files           []*File `json:"files,omitempty"`
+	NextPageToken   string  `json:"next_page_token"`
+	Version         string  `json:"version,omitempty"`
+	VersionOutdated bool    `json:"version_outdated,omitempty"`
+}
+
+// File is a basic element representing a single file object
+//
+// There are two types of download links,
+// 1) one from File.WebContentLink or File.Links.ApplicationOctetStream.URL and
+// 2) the other from File.Medias[].Link.URL.
+// Empirically, 2) is less restrictive to multiple concurrent range-requests
+// for a single file, i.e. supports for higher `--multi-thread-streams=N`.
+// However, it is not generally applicable as it is only for meadia.
+type File struct {
+	Apps              []*FileApp    `json:"apps,omitempty"`
+	Audit             *FileAudit    `json:"audit,omitempty"`
+	Collection        string        `json:"collection,omitempty"` // TODO
+	CreatedTime       Time          `json:"created_time,omitempty"`
+	DeleteTime        Time          `json:"delete_time,omitempty"`
+	FileCategory      string        `json:"file_category,omitempty"`
+	FileExtension     string        `json:"file_extension,omitempty"`
+	FolderType        string        `json:"folder_type,omitempty"`
+	Hash              string        `json:"hash,omitempty"` // sha1 but NOT a valid file hash. looks like a torrent hash
+	IconLink          string        `json:"icon_link,omitempty"`
+	ID                string        `json:"id,omitempty"`
+	Kind              string        `json:"kind,omitempty"` // "drive#file"
+	Links             *FileLinks    `json:"links,omitempty"`
+	Md5Checksum       string        `json:"md5_checksum,omitempty"`
+	Medias            []*Media      `json:"medias,omitempty"`
+	MimeType          string        `json:"mime_type,omitempty"`
+	ModifiedTime      Time          `json:"modified_time,omitempty"` // updated when renamed or moved
+	Name              string        `json:"name,omitempty"`
+	OriginalFileIndex int           `json:"original_file_index,omitempty"` // TODO
+	OriginalURL       string        `json:"original_url,omitempty"`
+	Params            *FileParams   `json:"params,omitempty"`
+	ParentID          string        `json:"parent_id,omitempty"`
+	Phase             string        `json:"phase,omitempty"`
+	Revision          int           `json:"revision,omitempty,string"`
+	Size              int64         `json:"size,omitempty,string"`
+	SortName          string        `json:"sort_name,omitempty"`
+	Space             string        `json:"space,omitempty"`
+	SpellName         []interface{} `json:"spell_name,omitempty"` // TODO maybe list of something?
+	Starred           bool          `json:"starred,omitempty"`
+	ThumbnailLink     string        `json:"thumbnail_link,omitempty"`
+	Trashed           bool          `json:"trashed,omitempty"`
+	UserID            string        `json:"user_id,omitempty"`
+	UserModifiedTime  Time          `json:"user_modified_time,omitempty"`
+	WebContentLink    string        `json:"web_content_link,omitempty"`
+	Writable          bool          `json:"writable,omitempty"`
+}
+
+// FileLinks includes links to file at backend
+type FileLinks struct {
+	ApplicationOctetStream *Link `json:"application/octet-stream,omitempty"`
+}
+
+// FileAudit contains audit information for the file
+type FileAudit struct {
+	Status  string `json:"status,omitempty"` // "STATUS_OK"
+	Message string `json:"message,omitempty"`
+	Title   string `json:"title,omitempty"`
+}
+
+// Media contains info about supported version of media, e.g. original, transcoded, etc
+type Media struct {
+	MediaID   string `json:"media_id,omitempty"`
+	MediaName string `json:"media_name,omitempty"`
+	Video     struct {
+		Height     int    `json:"height,omitempty"`
+		Width      int    `json:"width,omitempty"`
+		Duration   int64  `json:"duration,omitempty"`
+		BitRate    int    `json:"bit_rate,omitempty"`
+		FrameRate  int    `json:"frame_rate,omitempty"`
+		VideoCodec string `json:"video_codec,omitempty"`
+		AudioCodec string `json:"audio_codec,omitempty"`
+		VideoType  string `json:"video_type,omitempty"`
+	} `json:"video,omitempty"`
+	Link           *Link         `json:"link,omitempty"`
+	NeedMoreQuota  bool          `json:"need_more_quota,omitempty"`
+	VipTypes       []interface{} `json:"vip_types,omitempty"` // TODO maybe list of something?
+	RedirectLink   string        `json:"redirect_link,omitempty"`
+	IconLink       string        `json:"icon_link,omitempty"`
+	IsDefault      bool          `json:"is_default,omitempty"`
+	Priority       int           `json:"priority,omitempty"`
+	IsOrigin       bool          `json:"is_origin,omitempty"`
+	ResolutionName string        `json:"resolution_name,omitempty"`
+	IsVisible      bool          `json:"is_visible,omitempty"`
+	Category       string        `json:"category,omitempty"`
+}
+
+// FileParams includes parameters for instant open
+type FileParams struct {
+	Duration     int64  `json:"duration,omitempty,string"` // in seconds
+	Height       int    `json:"height,omitempty,string"`
+	Platform     string `json:"platform,omitempty"` // "Upload"
+	PlatformIcon string `json:"platform_icon,omitempty"`
+	URL          string `json:"url,omitempty"`
+	Width        int    `json:"width,omitempty,string"`
+}
+
+// FileApp includes parameters for instant open
+type FileApp struct {
+	ID            string        `json:"id,omitempty"`   // "decompress" for rar files
+	Name          string        `json:"name,omitempty"` // decompress" for rar files
+	Access        []interface{} `json:"access,omitempty"`
+	Link          string        `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
+	RedirectLink  string        `json:"redirect_link,omitempty"`
+	VipTypes      []interface{} `json:"vip_types,omitempty"`
+	NeedMoreQuota bool          `json:"need_more_quota,omitempty"`
+	IconLink      string        `json:"icon_link,omitempty"`
+	IsDefault     bool          `json:"is_default,omitempty"`
+	Params        struct {
+	} `json:"params,omitempty"` // TODO
+	CategoryIds []interface{} `json:"category_ids,omitempty"`
+	AdSceneType int           `json:"ad_scene_type,omitempty"`
+	Space       string        `json:"space,omitempty"`
+	Links       struct {
+	} `json:"links,omitempty"` // TODO
+}
+
+// ------------------------------------------------------------
+
+// TaskList contains a list of Task elements
+type TaskList struct {
+	Tasks         []*Task `json:"tasks,omitempty"` // "drive#task"
+	NextPageToken string  `json:"next_page_token"`
+	ExpiresIn     int     `json:"expires_in,omitempty"`
+}
+
+// Task is a basic element representing a single task such as offline download and upload
+type Task struct {
+	Kind              string        `json:"kind,omitempty"` // "drive#task"
+	ID                string        `json:"id,omitempty"`   // task id?
+	Name              string        `json:"name,omitempty"` // torrent name?
+	Type              string        `json:"type,omitempty"` // "offline"
+	UserID            string        `json:"user_id,omitempty"`
+	Statuses          []interface{} `json:"statuses,omitempty"`    // TODO
+	StatusSize        int           `json:"status_size,omitempty"` // TODO
+	Params            *TaskParams   `json:"params,omitempty"`      // TODO
+	FileID            string        `json:"file_id,omitempty"`
+	FileName          string        `json:"file_name,omitempty"`
+	FileSize          string        `json:"file_size,omitempty"`
+	Message           string        `json:"message,omitempty"` // e.g. "Saving"
+	CreatedTime       Time          `json:"created_time,omitempty"`
+	UpdatedTime       Time          `json:"updated_time,omitempty"`
+	ThirdTaskID       string        `json:"third_task_id,omitempty"` // TODO
+	Phase             string        `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
+	Progress          int           `json:"progress,omitempty"`
+	IconLink          string        `json:"icon_link,omitempty"`
+	Callback          string        `json:"callback,omitempty"`
+	ReferenceResource interface{}   `json:"reference_resource,omitempty"` // TODO
+	Space             string        `json:"space,omitempty"`
+}
+
+// TaskParams includes parameters informing status of Task
+type TaskParams struct {
+	Age          string `json:"age,omitempty"`
+	PredictSpeed string `json:"predict_speed,omitempty"`
+	PredictType  string `json:"predict_type,omitempty"`
+	URL          string `json:"url,omitempty"`
+}
+
+// Form contains parameters for upload by multipart/form-data
+type Form struct {
+	Headers    struct{} `json:"headers"`
+	Kind       string   `json:"kind"`   // "drive#form"
+	Method     string   `json:"method"` // "POST"
+	MultiParts struct {
+		OSSAccessKeyID string `json:"OSSAccessKeyId"`
+		Signature      string `json:"Signature"`
+		Callback       string `json:"callback"`
+		Key            string `json:"key"`
+		Policy         string `json:"policy"`
+		XUserData      string `json:"x:user_data"`
+	} `json:"multi_parts"`
+	URL string `json:"url"`
+}
+
+// Resumable contains parameters for upload by resumable
+type Resumable struct {
+	Kind     string           `json:"kind,omitempty"`     // "drive#resumable"
+	Provider string           `json:"provider,omitempty"` // e.g. "PROVIDER_ALIYUN"
+	Params   *ResumableParams `json:"params,omitempty"`
+}
+
+// ResumableParams specifies resumable paramegers
+type ResumableParams struct {
+	AccessKeyID     string `json:"access_key_id,omitempty"`
+	AccessKeySecret string `json:"access_key_secret,omitempty"`
+	Bucket          string `json:"bucket,omitempty"`
+	Endpoint        string `json:"endpoint,omitempty"`
+	Expiration      Time   `json:"expiration,omitempty"`
+	Key             string `json:"key,omitempty"`
+	SecurityToken   string `json:"security_token,omitempty"`
+}
+
+// FileInArchive is a basic element in archive
+type FileInArchive struct {
+	Index    int    `json:"index,omitempty"`
+	Filename string `json:"filename,omitempty"`
+	Filesize string `json:"filesize,omitempty"`
+	MimeType string `json:"mime_type,omitempty"`
+	Gcid     string `json:"gcid,omitempty"`
+	Kind     string `json:"kind,omitempty"`
+	IconLink string `json:"icon_link,omitempty"`
+	Path     string `json:"path,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// NewFile is a response to RequestNewFile
+type NewFile struct {
+	File       *File      `json:"file,omitempty"`
+	Form       *Form      `json:"form,omitempty"`
+	Resumable  *Resumable `json:"resumable,omitempty"`
+	Task       *Task      `json:"task,omitempty"`        // null in this case
+	UploadType string     `json:"upload_type,omitempty"` // "UPLOAD_TYPE_FORM" or "UPLOAD_TYPE_RESUMABLE"
+}
+
+// NewTask is a response to RequestNewTask
+type NewTask struct {
+	UploadType string `json:"upload_type,omitempty"` // "UPLOAD_TYPE_URL"
+	File       *File  `json:"file,omitempty"`        // null in this case
+	Task       *Task  `json:"task,omitempty"`
+	URL        *URL   `json:"url,omitempty"` // {"kind": "upload#url"}
+}
+
+// About informs drive status
+type About struct {
+	Kind      string `json:"kind,omitempty"` // "drive#about"
+	Quota     *Quota `json:"quota,omitempty"`
+	ExpiresAt string `json:"expires_at,omitempty"`
+	Quotas    struct {
+	} `json:"quotas,omitempty"` // maybe []*Quota?
+}
+
+// Quota informs drive quota
+type Quota struct {
+	Kind           string `json:"kind,omitempty"`                  // "drive#quota"
+	Limit          int64  `json:"limit,omitempty,string"`          // limit in bytes
+	Usage          int64  `json:"usage,omitempty,string"`          // bytes in use
+	UsageInTrash   int64  `json:"usage_in_trash,omitempty,string"` // bytes in trash but this seems not working
+	PlayTimesLimit string `json:"play_times_limit,omitempty"`      // maybe in seconds
+	PlayTimesUsage string `json:"play_times_usage,omitempty"`      // maybe in seconds
+}
+
+// Share is a response to RequestShare
+//
+// used in PublicLink()
+type Share struct {
+	ShareID   string `json:"share_id,omitempty"`
+	ShareURL  string `json:"share_url,omitempty"`
+	PassCode  string `json:"pass_code,omitempty"`
+	ShareText string `json:"share_text,omitempty"`
+}
+
+// User contains user account information
+//
+// GET https://user.mypikpak.com/v1/user/me
+type User struct {
+	Sub               string          `json:"sub,omitempty"`       // userid for internal use
+	Name              string          `json:"name,omitempty"`      // Username
+	Picture           string          `json:"picture,omitempty"`   // URL to Avatar image
+	Email             string          `json:"email,omitempty"`     // redacted email address
+	Providers         *[]UserProvider `json:"providers,omitempty"` // OAuth provider
+	PhoneNumber       string          `json:"phone_number,omitempty"`
+	Password          string          `json:"password,omitempty"` // "SET" if configured
+	Status            string          `json:"status,omitempty"`   // "ACTIVE"
+	CreatedAt         Time            `json:"created_at,omitempty"`
+	PasswordUpdatedAt Time            `json:"password_updated_at,omitempty"`
+}
+
+// UserProvider details third-party authentication
+type UserProvider struct {
+	ID             string `json:"id,omitempty"` // e.g. "google.com"
+	ProviderUserID string `json:"provider_user_id,omitempty"`
+	Name           string `json:"name,omitempty"` // username
+}
+
+// VIP includes subscription details about premium account
+//
+// GET https://api-drive.mypikpak.com/drive/v1/privilege/vip
+type VIP struct {
+	Result      string `json:"result,omitempty"` // "ACCEPTED"
+	Message     string `json:"message,omitempty"`
+	RedirectURI string `json:"redirect_uri,omitempty"`
+	Data        struct {
+		Expire Time   `json:"expire,omitempty"`
+		Status string `json:"status,omitempty"`  // "invalid" or "ok"
+		Type   string `json:"type,omitempty"`    // "novip" or "platinum"
+		UserID string `json:"user_id,omitempty"` // same as User.Sub
+	} `json:"data,omitempty"`
+}
+
+// DecompressResult is a response to RequestDecompress
+type DecompressResult struct {
+	Status       string `json:"status,omitempty"` // "OK"
+	StatusText   string `json:"status_text,omitempty"`
+	TaskID       string `json:"task_id,omitempty"`   // same as File.Id
+	FilesNum     int    `json:"files_num,omitempty"` // number of files in archive
+	RedirectLink string `json:"redirect_link,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// RequestShare is to request for file share
+type RequestShare struct {
+	FileIds        []string `json:"file_ids,omitempty"`
+	ShareTo        string   `json:"share_to,omitempty"`         // "publiclink",
+	ExpirationDays int      `json:"expiration_days,omitempty"`  // -1 = 'forever'
+	PassCodeOption string   `json:"pass_code_option,omitempty"` // "NOT_REQUIRED"
+}
+
+// RequestBatch is to request for batch actions
+type RequestBatch struct {
+	Ids []string          `json:"ids,omitempty"`
+	To  map[string]string `json:"to,omitempty"`
+}
+
+// RequestNewFile is to request for creating a new `drive#folder` or `drive#file`
+type RequestNewFile struct {
+	// always required
+	Kind       string `json:"kind"` // "drive#folder" or "drive#file"
+	Name       string `json:"name"`
+	ParentID   string `json:"parent_id"`
+	FolderType string `json:"folder_type"`
+	// only when uploading a new file
+	Hash       string            `json:"hash,omitempty"`      // sha1sum
+	Resumable  map[string]string `json:"resumable,omitempty"` // {"provider": "PROVIDER_ALIYUN"}
+	Size       int64             `json:"size,omitempty"`
+	UploadType string            `json:"upload_type,omitempty"` // "UPLOAD_TYPE_FORM" or "UPLOAD_TYPE_RESUMABLE"
+}
+
+// RequestNewTask is to request for creating a new task like offline downloads
+//
+// Name and ParentID can be left empty.
+type RequestNewTask struct {
+	Kind       string `json:"kind,omitempty"` // "drive#file"
+	Name       string `json:"name,omitempty"`
+	ParentID   string `json:"parent_id,omitempty"`
+	UploadType string `json:"upload_type,omitempty"` // "UPLOAD_TYPE_URL"
+	URL        *URL   `json:"url,omitempty"`         // {"url": downloadUrl}
+	FolderType string `json:"folder_type,omitempty"` // "" if parent_id else "DOWNLOAD"
+}
+
+// RequestDecompress is to request for decompress of archive files
+type RequestDecompress struct {
+	Gcid          string           `json:"gcid,omitempty"`     // same as File.Hash
+	Password      string           `json:"password,omitempty"` // ""
+	FileID        string           `json:"file_id,omitempty"`
+	Files         []*FileInArchive `json:"files,omitempty"` // can request selected files to be decompressed
+	DefaultParent bool             `json:"default_parent,omitempty"`
+}
+
+// ------------------------------------------------------------
+
+// NOT implemented YET
+
+// RequestArchiveFileList is to request for a list of files in archive
+//
+// POST https://api-drive.mypikpak.com/decompress/v1/list
+type RequestArchiveFileList struct {
+	Gcid     string `json:"gcid,omitempty"`     // same as api.File.Hash
+	Path     string `json:"path,omitempty"`     // "" by default
+	Password string `json:"password,omitempty"` // "" by default
+	FileID   string `json:"file_id,omitempty"`
+}
+
+// ArchiveFileList is a response to RequestArchiveFileList
+type ArchiveFileList struct {
+	Status      string           `json:"status,omitempty"`       // "OK"
+	StatusText  string           `json:"status_text,omitempty"`  // ""
+	TaskID      string           `json:"task_id,omitempty"`      // ""
+	CurrentPath string           `json:"current_path,omitempty"` // ""
+	Title       string           `json:"title,omitempty"`
+	FileSize    int64            `json:"file_size,omitempty"`
+	Gcid        string           `json:"gcid,omitempty"` // same as File.Hash
+	Files       []*FileInArchive `json:"files,omitempty"`
+}

backend/pikpak/helper.go

@@ -0,0 +1,253 @@
+package pikpak
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	"github.com/rclone/rclone/backend/pikpak/api"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Globals
+const (
+	cachePrefix = "rclone-pikpak-sha1sum-"
+)
+
+// requestDecompress requests decompress of compressed files
+func (f *Fs) requestDecompress(ctx context.Context, file *api.File, password string) (info *api.DecompressResult, err error) {
+	req := &api.RequestDecompress{
+		Gcid:          file.Hash,
+		Password:      password,
+		FileID:        file.ID,
+		Files:         []*api.FileInArchive{},
+		DefaultParent: true,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/decompress/v1/decompress",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getUserInfo gets UserInfo from API
+func (f *Fs) getUserInfo(ctx context.Context) (info *api.User, err error) {
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: "https://user.mypikpak.com/v1/user/me",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get userinfo: %w", err)
+	}
+	return
+}
+
+// getVIPInfo gets VIPInfo from API
+func (f *Fs) getVIPInfo(ctx context.Context) (info *api.VIP, err error) {
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: "https://api-drive.mypikpak.com/drive/v1/privilege/vip",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get vip info: %w", err)
+	}
+	return
+}
+
+// requestBatchAction requests batch actions to API
+//
+// action can be one of batch{Copy,Delete,Trash,Untrash}
+func (f *Fs) requestBatchAction(ctx context.Context, action string, req *api.RequestBatch) (err error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/drive/v1/files:" + action,
+		NoResponse: true, // Only returns `{"task_id":""}
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, nil)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("batch action %q failed: %w", action, err)
+	}
+	return nil
+}
+
+// requestNewTask requests a new api.NewTask and returns api.Task
+func (f *Fs) requestNewTask(ctx context.Context, req *api.RequestNewTask) (info *api.Task, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/files",
+	}
+	var newTask api.NewTask
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &newTask)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return newTask.Task, nil
+}
+
+// requestNewFile requests a new api.NewFile and returns api.File
+func (f *Fs) requestNewFile(ctx context.Context, req *api.RequestNewFile) (info *api.NewFile, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/files",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getFile gets api.File from API for the ID passed
+// and returns rich information containing additional fields below
+// * web_content_link
+// * thumbnail_link
+// * links
+// * medias
+func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/drive/v1/files/" + ID,
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		if err == nil && info.Phase != api.PhaseTypeComplete {
+			// could be pending right after file is created/uploaded.
+			return true, errors.New("not PHASE_TYPE_COMPLETE")
+		}
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// patchFile updates attributes of the file by ID
+//
+// currently known patchable fields are
+// * name
+func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api.File, err error) {
+	opts := rest.Opts{
+		Method: "PATCH",
+		Path:   "/drive/v1/files/" + ID,
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// getAbout gets drive#quota information from server
+func (f *Fs) getAbout(ctx context.Context) (info *api.About, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/drive/v1/about",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// requestShare returns information about ssharable links
+func (f *Fs) requestShare(ctx context.Context, req *api.RequestShare) (info *api.Share, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/drive/v1/share",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.rst.CallJSON(ctx, &opts, &req, &info)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	return
+}
+
+// Read the sha1 of in returning a reader which will read the same contents
+//
+// The cleanup function should be called when out is finished with
+// regardless of whether this function returned an error or not.
+func readSHA1(in io.Reader, size, threshold int64) (sha1sum string, out io.Reader, cleanup func(), err error) {
+	// we need an SHA1
+	hash := sha1.New()
+	// use the teeReader to write to the local file AND calculate the SHA1 while doing so
+	teeReader := io.TeeReader(in, hash)
+
+	// nothing to clean up by default
+	cleanup = func() {}
+
+	// don't cache small files on disk to reduce wear of the disk
+	if size > threshold {
+		var tempFile *os.File
+
+		// create the cache file
+		tempFile, err = os.CreateTemp("", cachePrefix)
+		if err != nil {
+			return
+		}
+
+		_ = os.Remove(tempFile.Name()) // Delete the file - may not work on Windows
+
+		// clean up the file after we are done downloading
+		cleanup = func() {
+			// the file should normally already be close, but just to make sure
+			_ = tempFile.Close()
+			_ = os.Remove(tempFile.Name()) // delete the cache file after we are done - may be deleted already
+		}
+
+		// copy the ENTIRE file to disc and calculate the SHA1 in the process
+		if _, err = io.Copy(tempFile, teeReader); err != nil {
+			return
+		}
+		// jump to the start of the local file so we can pass it along
+		if _, err = tempFile.Seek(0, 0); err != nil {
+			return
+		}
+
+		// replace the already read source with a reader of our cached file
+		out = tempFile
+	} else {
+		// that's a small file, just read it into memory
+		var inData []byte
+		inData, err = io.ReadAll(teeReader)
+		if err != nil {
+			return
+		}
+
+		// set the reader to our read memory block
+		out = bytes.NewReader(inData)
+	}
+	return hex.EncodeToString(hash.Sum(nil)), out, cleanup, nil
+}

backend/pikpak/pikpak_test.go

@@ -0,0 +1,17 @@
+// Test PikPak filesystem interface
+package pikpak_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/pikpak"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestPikPak:",
+		NilObject:  (*pikpak.Object)(nil),
+	})
+}

backend/s3/s3.go

@@ -1,4 +1,4 @@
-// Package s3 provides an interface to Amazon S3 oject storage
+// Package s3 provides an interface to Amazon S3 object storage
 package s3
 
 //go:generate go run gen_setfrom.go -o setfrom.go
@@ -66,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -109,6 +109,9 @@ func init() {
 			}, {
 				Value: "Dreamhost",
 				Help:  "Dreamhost DreamObjects",
+			}, {
+				Value: "GCS",
+				Help:  "Google Cloud Storage",
 			}, {
 				Value: "HuaweiOBS",
 				Help:  "Huawei Object Storage Service",
@@ -934,6 +937,14 @@ func init() {
 				Value: "s3.eu-central-1.stackpathstorage.com",
 				Help:  "EU Endpoint",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Google Cloud Storage.",
+			Provider: "GCS",
+			Examples: []fs.OptionExample{{
+				Value: "https://storage.googleapis.com",
+				Help:  "Google Cloud Storage endpoint",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for Storj Gateway.",
@@ -1098,7 +1109,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
+			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -2258,6 +2269,24 @@ will decompress the object on the fly.
 If this is set to unset (the default) then rclone will choose
 according to the provider setting what to apply, but you can override
 rclone's choice here.
+`, "|", "`"),
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "use_accept_encoding_gzip",
+			Help: strings.ReplaceAll(`Whether to send |Accept-Encoding: gzip| header.
+
+By default, rclone will append |Accept-Encoding: gzip| to the request to download
+compressed objects whenever possible.
+
+However some providers such as Google Cloud Storage may alter the HTTP headers, breaking
+the signature of the request.
+
+A symptom of this would be receiving errors like
+
+	SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided.
+
+In this case, you might want to try disabling this option.
 `, "|", "`"),
 			Default:  fs.Tristate{},
 			Advanced: true,
@@ -2399,6 +2428,7 @@ type Options struct {
 	VersionAt             fs.Time              `config:"version_at"`
 	Decompress            bool                 `config:"decompress"`
 	MightGzip             fs.Tristate          `config:"might_gzip"`
+	UseAcceptEncodingGzip fs.Tristate          `config:"use_accept_encoding_gzip"`
 	NoSystemMetadata      bool                 `config:"no_system_metadata"`
 }
 
@@ -2791,11 +2821,12 @@ func setEndpointValueForIDriveE2(m configmap.Mapper) (err error) {
 // These should be differences from AWS S3
 func setQuirks(opt *Options) {
 	var (
-		listObjectsV2     = true
-		virtualHostStyle  = true
-		urlEncodeListings = true
-		useMultipartEtag  = true
-		mightGzip         = true // assume all providers might gzip until proven otherwise
+		listObjectsV2         = true
+		virtualHostStyle      = true
+		urlEncodeListings     = true
+		useMultipartEtag      = true
+		useAcceptEncodingGzip = true
+		mightGzip             = true // assume all providers might gzip until proven otherwise
 	)
 	switch opt.Provider {
 	case "AWS":
@@ -2880,6 +2911,10 @@ func setQuirks(opt *Options) {
 	case "Qiniu":
 		useMultipartEtag = false
 		urlEncodeListings = false
+	case "GCS":
+		// Google break request Signature by mutating accept-encoding HTTP header
+		// https://github.com/rclone/rclone/issues/6670
+		useAcceptEncodingGzip = false
 	case "Other":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -2924,6 +2959,12 @@ func setQuirks(opt *Options) {
 		opt.MightGzip.Valid = true
 		opt.MightGzip.Value = mightGzip
 	}
+
+	// set UseAcceptEncodingGzip if not manually set
+	if !opt.UseAcceptEncodingGzip.Valid {
+		opt.UseAcceptEncodingGzip.Valid = true
+		opt.UseAcceptEncodingGzip.Value = useAcceptEncodingGzip
+	}
 }
 
 // setRoot changes the root of the Fs
@@ -2957,7 +2998,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, fmt.Errorf("s3: upload cutoff: %w", err)
 	}
 	if opt.Versions && opt.VersionAt.IsSet() {
-		return nil, errors.New("s3: cant use --s3-versions and --s3-version-at at the same time")
+		return nil, errors.New("s3: can't use --s3-versions and --s3-version-at at the same time")
 	}
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
@@ -3078,6 +3119,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 	err = f.list(ctx, listOpt{
 		bucket:       bucket,
 		directory:    bucketPath,
+		prefix:       f.rootDirectory,
 		recurse:      true,
 		withVersions: f.opt.Versions,
 		findFile:     true,
@@ -3483,10 +3525,10 @@ type listOpt struct {
 // list lists the objects into the function supplied with the opt
 // supplied.
 func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
+	if opt.prefix != "" {
+		opt.prefix += "/"
+	}
 	if !opt.findFile {
-		if opt.prefix != "" {
-			opt.prefix += "/"
-		}
 		if opt.directory != "" {
 			opt.directory += "/"
 		}
@@ -4962,7 +5004,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	// Override the automatic decompression in the transport to
 	// download compressed files as-is
-	httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
+	if o.fs.opt.UseAcceptEncodingGzip.Value {
+		httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip")
+	}
 
 	for _, option := range options {
 		switch option.(type) {
@@ -5078,7 +5122,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	uid := cout.UploadId
 
+	uploadCtx, cancel := context.WithCancel(ctx)
 	defer atexit.OnError(&err, func() {
+		cancel()
 		if o.fs.opt.LeavePartsOnError {
 			return
 		}
@@ -5098,7 +5144,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	})()
 
 	var (
-		g, gCtx  = errgroup.WithContext(ctx)
+		g, gCtx  = errgroup.WithContext(uploadCtx)
 		finished = false
 		partsMu  sync.Mutex // to protect parts
 		parts    []*s3.CompletedPart
@@ -5180,7 +5226,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 				uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
 				if err != nil {
 					if partNum <= int64(concurrency) {
-						return f.shouldRetry(ctx, err)
+						return f.shouldRetry(gCtx, err)
 					}
 					// retry all chunks once have done the first batch
 					return true, err
@@ -5212,7 +5258,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 
 	var resp *s3.CompleteMultipartUploadOutput
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
+		resp, err = f.c.CompleteMultipartUploadWithContext(uploadCtx, &s3.CompleteMultipartUploadInput{
 			Bucket: req.Bucket,
 			Key:    req.Key,
 			MultipartUpload: &s3.CompletedMultipartUpload{
@@ -5221,7 +5267,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 			RequestPayer: req.RequestPayer,
 			UploadId:     uid,
 		})
-		return f.shouldRetry(ctx, err)
+		return f.shouldRetry(uploadCtx, err)
 	})
 	if err != nil {
 		return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)

backend/s3/s3_internal_test.go

@@ -6,12 +6,15 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"path"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -250,7 +253,8 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// Create an object
-	const fileName = "test-versions.txt"
+	const dirName = "versions"
+	const fileName = dirName + "/" + "test-versions.txt"
 	contents := random.String(100)
 	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
@@ -280,11 +284,12 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 		}()
 
 		// Read the contents
-		entries, err := f.List(ctx, "")
+		entries, err := f.List(ctx, dirName)
 		require.NoError(t, err)
 		tests := 0
 		var fileNameVersion string
 		for _, entry := range entries {
+			t.Log(entry)
 			remote := entry.Remote()
 			if remote == fileName {
 				t.Run("ReadCurrent", func(t *testing.T) {
@@ -309,6 +314,18 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 			require.NotNil(t, o)
 			assert.Equal(t, int64(100), o.Size(), o.Remote())
 		})
+
+		// Check we can make a NewFs from that object with a version suffix
+		t.Run("NewFs", func(t *testing.T) {
+			newPath := path.Join(fs.ConfigString(f), fileNameVersion)
+			// Make sure --s3-versions is set in the config of the new remote
+			confPath := strings.Replace(newPath, ":", ",versions:", 1)
+			fNew, err := cache.Get(ctx, confPath)
+			// This should return pointing to a file
+			assert.Equal(t, fs.ErrorIsFile, err)
+			// With the directory the directory above
+			assert.Equal(t, dirName, path.Base(fs.ConfigString(fNew)))
+		})
 	})
 
 	t.Run("VersionAt", func(t *testing.T) {

backend/seafile/webapi.go

@@ -953,11 +953,9 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
 	return nil
 }
 
-// === API v2 from the official documentation, but that have been replaced by the much better v2.1 (undocumented as of Apr 2020)
-// === getDirectoryEntriesAPIv2 is needed to keep compatibility with seafile v6,
-// === the others can probably be removed after the API v2.1 is documented
-
 func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath string) ([]api.DirEntry, error) {
+	// API v2 from the official documentation, but that have been replaced by the much better v2.1 (undocumented as of Apr 2020)
+	// getDirectoryEntriesAPIv2 is needed to keep compatibility with seafile v6.
 	// API Documentation
 	// https://download.seafile.com/published/web-api/v2.1/directories.md#user-content-List%20Items%20in%20Directory
 	if libraryID == "" {
@@ -1001,95 +999,3 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
 	}
 	return result, nil
 }
-
-func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibraryID, dstPath string) (*api.FileInfo, error) {
-	// API Documentation
-	// https://download.seafile.com/published/web-api/v2.1/file.md#user-content-Copy%20File
-	if srcLibraryID == "" || dstLibraryID == "" {
-		return nil, errors.New("libraryID and/or file path argument(s) missing")
-	}
-	srcPath = path.Join("/", srcPath)
-	dstPath = path.Join("/", dstPath)
-
-	// Older API does not seem to accept JSON input here either
-	postParameters := url.Values{
-		"operation": {"copy"},
-		"dst_repo":  {dstLibraryID},
-		"dst_dir":   {f.opt.Enc.FromStandardPath(dstPath)},
-	}
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        APIv20 + srcLibraryID + "/file/",
-		Parameters:  url.Values{"p": {f.opt.Enc.FromStandardPath(srcPath)}},
-		ContentType: "application/x-www-form-urlencoded",
-		Body:        bytes.NewBuffer([]byte(postParameters.Encode())),
-	}
-	result := &api.FileInfo{}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		if resp != nil {
-			if resp.StatusCode == 401 || resp.StatusCode == 403 {
-				return nil, fs.ErrorPermissionDenied
-			}
-		}
-		return nil, fmt.Errorf("failed to copy file %s:'%s' to %s:'%s': %w", srcLibraryID, srcPath, dstLibraryID, dstPath, err)
-	}
-	err = rest.DecodeJSON(resp, &result)
-	if err != nil {
-		return nil, err
-	}
-	return f.decodeFileInfo(result), nil
-}
-
-func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname string) error {
-	// API Documentation
-	// https://download.seafile.com/published/web-api/v2.1/file.md#user-content-Rename%20File
-	if libraryID == "" || newname == "" {
-		return errors.New("libraryID and/or file path argument(s) missing")
-	}
-	filePath = path.Join("/", filePath)
-
-	// No luck with JSON input with the older api2
-	postParameters := url.Values{
-		"operation": {"rename"},
-		"reloaddir": {"true"}, // This is an undocumented trick to avoid an http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
-		"newname":   {f.opt.Enc.FromStandardName(newname)},
-	}
-
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        APIv20 + libraryID + "/file/",
-		Parameters:  url.Values{"p": {f.opt.Enc.FromStandardPath(filePath)}},
-		ContentType: "application/x-www-form-urlencoded",
-		Body:        bytes.NewBuffer([]byte(postParameters.Encode())),
-		NoRedirect:  true,
-		NoResponse:  true,
-	}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		if resp != nil {
-			if resp.StatusCode == 301 {
-				// This is the normal response from the server
-				return nil
-			}
-			if resp.StatusCode == 401 || resp.StatusCode == 403 {
-				return fs.ErrorPermissionDenied
-			}
-			if resp.StatusCode == 404 {
-				return fs.ErrorObjectNotFound
-			}
-		}
-		return fmt.Errorf("failed to rename file: %w", err)
-	}
-	return nil
-}

backend/sftp/sftp.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	iofs "io/fs"
 	"os"
 	"path"
 	"regexp"
@@ -323,7 +324,7 @@ Pass multiple variables space separated, eg
 
     VAR1=value VAR2=value
 
-and pass variables with spaces in in quotes, eg
+and pass variables with spaces in quotes, eg
 
     "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
 
@@ -367,6 +368,20 @@ At least one must match with server configuration. This can be checked for examp
 Example:
 
     umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
+`,
+			Advanced: true,
+		}, {
+			Name:    "host_key_algorithms",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of host key algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q HostKeyAlgorithms.
+
+Note: This can affect the outcome of key negotiation with the server even if server host key validation is not enabled.
+
+Example:
+
+    ssh-ed25519 ssh-rsa ssh-dss
 `,
 			Advanced: true,
 		}},
@@ -407,6 +422,7 @@ type Options struct {
 	Ciphers                 fs.SpaceSepList `config:"ciphers"`
 	KeyExchange             fs.SpaceSepList `config:"key_exchange"`
 	MACs                    fs.SpaceSepList `config:"macs"`
+	HostKeyAlgorithms       fs.SpaceSepList `config:"host_key_algorithms"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -739,6 +755,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ClientVersion:   "SSH-2.0-" + f.ci.UserAgent,
 	}
 
+	if len(opt.HostKeyAlgorithms) != 0 {
+		sshConfig.HostKeyAlgorithms = []string(opt.HostKeyAlgorithms)
+	}
+
 	if opt.KnownHostsFile != "" {
 		hostcallback, err := knownhosts.New(env.ShellExpand(opt.KnownHostsFile))
 		if err != nil {
@@ -782,10 +802,32 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("couldn't read ssh agent signers: %w", err)
 		}
 		if keyFile != "" {
+			// If `opt.KeyUseAgent` is false, then it's expected that `opt.KeyFile` contains the private key
+			// and `${opt.KeyFile}.pub` contains the public key.
+			//
+			// If `opt.KeyUseAgent` is true, then it's expected that `opt.KeyFile` contains the public key.
+			// This is how it works with openssh; the `IdentityFile` in openssh config points to the public key.
+			// It's not necessary to specify the public key explicitly when using ssh-agent, since openssh and rclone
+			// will try all the keys they find in the ssh-agent until they find one that works. But just like
+			// `IdentityFile` is used in openssh config to limit the search to one specific key, so does
+			// `opt.KeyFile` in rclone config limit the search to that specific key.
+			//
+			// However, previous versions of rclone would always expect to find the public key in
+			// `${opt.KeyFile}.pub` even if `opt.KeyUseAgent` was true. So for the sake of backward compatibility
+			// we still first attempt to read the public key from `${opt.KeyFile}.pub`. But if it fails with
+			// an `fs.ErrNotExist` then we also try to read the public key from `opt.KeyFile`.
 			pubBytes, err := os.ReadFile(keyFile + ".pub")
 			if err != nil {
-				return nil, fmt.Errorf("failed to read public key file: %w", err)
+				if errors.Is(err, iofs.ErrNotExist) && opt.KeyUseAgent {
+					pubBytes, err = os.ReadFile(keyFile)
+					if err != nil {
+						return nil, fmt.Errorf("failed to read public key file: %w", err)
+					}
+				} else {
+					return nil, fmt.Errorf("failed to read public key file: %w", err)
+				}
 			}
+
 			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse public key file: %w", err)
@@ -807,8 +849,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	}
 
-	// Load key file if specified
-	if keyFile != "" || opt.KeyPem != "" {
+	// Load key file as a private key, if specified. This is only needed when not using an ssh agent.
+	if (keyFile != "" && !opt.KeyUseAgent) || opt.KeyPem != "" {
 		var key []byte
 		if opt.KeyPem == "" {
 			key, err = os.ReadFile(keyFile)

backend/smb/connpool.go

@@ -34,9 +34,10 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{
 		Initiator: &smb2.NTLMInitiator{
-			User:     f.opt.User,
-			Password: pass,
-			Domain:   f.opt.Domain,
+			User:      f.opt.User,
+			Password:  pass,
+			Domain:    f.opt.Domain,
+			TargetSPN: f.opt.SPN,
 		},
 	}
 
@@ -105,9 +106,9 @@ func (f *Fs) getSessions() int32 {
 func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) {
 	// As we are pooling these connections we need to decouple
 	// them from the current context
-	ctx = context.Background()
+	bgCtx := context.Background()
 
-	c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port)
+	c, err = f.dial(bgCtx, "tcp", f.opt.Host+":"+f.opt.Port)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SMB: %w", err)
 	}
@@ -118,7 +119,7 @@ func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err erro
 			_ = c.smbSession.Logoff()
 			return nil, fmt.Errorf("couldn't initialize SMB: %w", err)
 		}
-		c.smbShare = c.smbShare.WithContext(ctx)
+		c.smbShare = c.smbShare.WithContext(bgCtx)
 	}
 	return c, nil
 }

backend/smb/smb.go

@@ -60,6 +60,17 @@ func init() {
 			Name:    "domain",
 			Help:    "Domain name for NTLM authentication.",
 			Default: "WORKGROUP",
+		}, {
+			Name: "spn",
+			Help: `Service principal name.
+
+Rclone presents this name to the server. Some servers use this as further
+authentication, and it often needs to be set for clusters. For example:
+
+    cifs/remotehost:1020
+
+Leave blank if not sure.
+`,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -109,6 +120,7 @@ type Options struct {
 	User            string      `config:"user"`
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
+	SPN             string      `config:"spn"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`

backend/swift/swift.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -1328,23 +1327,6 @@ func (o *Object) removeSegmentsLargeObject(ctx context.Context, containerSegment
 	return nil
 }
 
-func (o *Object) getSegmentsDlo(ctx context.Context) (segmentsContainer string, prefix string, err error) {
-	if err = o.readMetaData(ctx); err != nil {
-		return
-	}
-	dirManifest := o.headers["X-Object-Manifest"]
-	dirManifest, err = url.PathUnescape(dirManifest)
-	if err != nil {
-		return
-	}
-	delimiter := strings.Index(dirManifest, "/")
-	if len(dirManifest) == 0 || delimiter < 0 {
-		err = errors.New("missing or wrong structure of manifest of Dynamic large object")
-		return
-	}
-	return dirManifest[:delimiter], dirManifest[delimiter+1:], nil
-}
-
 // urlEncode encodes a string so that it is a valid URL
 //
 // We don't use any of Go's standard methods as we need `/` not

backend/union/union.go

@@ -756,14 +756,6 @@ func (f *Fs) create(ctx context.Context, path string) ([]*upstream.Fs, error) {
 	return f.createPolicy.Create(ctx, f.upstreams, path)
 }
 
-func (f *Fs) createEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
-	return f.createPolicy.CreateEntries(entries...)
-}
-
-func (f *Fs) search(ctx context.Context, path string) (*upstream.Fs, error) {
-	return f.searchPolicy.Search(ctx, f.upstreams, path)
-}
-
 func (f *Fs) searchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	return f.searchPolicy.SearchEntries(entries...)
 }

backend/uptobox/uptobox.go

@@ -214,7 +214,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 
 	client := fshttp.NewClient(ctx)
 	f.srv = rest.NewClient(client).SetRoot(apiBaseURL)
-	f.IDRegexp = regexp.MustCompile(`https://uptobox\.com/([a-zA-Z0-9]+)`)
+	f.IDRegexp = regexp.MustCompile(`^https://uptobox\.com/([a-zA-Z0-9]+)`)
 
 	_, err = f.readMetaDataForPath(ctx, f.dirPath(""), &api.MetadataRequestOptions{Limit: 10})
 	if err != nil {

backend/webdav/api/types.go

@@ -75,6 +75,7 @@ type Prop struct {
 	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
 	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
+	MESha1Hex    *string   `xml:"ME: prop>sha1hex,omitempty"` // Fastmail-specific sha1 checksum
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -102,22 +103,27 @@ func (p *Prop) StatusOK() bool {
 
 // Hashes returns a map of all checksums - may be nil
 func (p *Prop) Hashes() (hashes map[hash.Type]string) {
-	if len(p.Checksums) == 0 {
-		return nil
-	}
-	hashes = make(map[hash.Type]string)
-	for _, checksums := range p.Checksums {
-		checksums = strings.ToLower(checksums)
-		for _, checksum := range strings.Split(checksums, " ") {
-			switch {
-			case strings.HasPrefix(checksum, "sha1:"):
-				hashes[hash.SHA1] = checksum[5:]
-			case strings.HasPrefix(checksum, "md5:"):
-				hashes[hash.MD5] = checksum[4:]
+	if len(p.Checksums) > 0 {
+		hashes = make(map[hash.Type]string)
+		for _, checksums := range p.Checksums {
+			checksums = strings.ToLower(checksums)
+			for _, checksum := range strings.Split(checksums, " ") {
+				switch {
+				case strings.HasPrefix(checksum, "sha1:"):
+					hashes[hash.SHA1] = checksum[5:]
+				case strings.HasPrefix(checksum, "md5:"):
+					hashes[hash.MD5] = checksum[4:]
+				}
 			}
 		}
+		return hashes
+	} else if p.MESha1Hex != nil {
+		hashes = make(map[hash.Type]string)
+		hashes[hash.SHA1] = *p.MESha1Hex
+		return hashes
+	} else {
+		return nil
 	}
-	return hashes
 }
 
 // PropValue is a tagged name and value

backend/webdav/chunking.go

@@ -0,0 +1,215 @@
+package webdav
+
+/*
+	chunked update for Nextcloud
+	see https://docs.nextcloud.com/server/20/developer_manual/client_apis/WebDAV/chunking.html
+*/
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+func (f *Fs) shouldRetryChunkMerge(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	// Not found. Can be returned by NextCloud when merging chunks of an upload.
+	if resp != nil && resp.StatusCode == 404 {
+		return true, err
+	}
+
+	// 423 LOCKED
+	if resp != nil && resp.StatusCode == 423 {
+		return false, fmt.Errorf("merging the uploaded chunks failed with 423 LOCKED. This usually happens when the chunks merging is still in progress on NextCloud, but it may also indicate a failed transfer: %w", err)
+	}
+
+	return f.shouldRetry(ctx, resp, err)
+}
+
+// set the chunk size for testing
+func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	return
+}
+
+func (f *Fs) getChunksUploadURL() string {
+	return strings.Replace(f.endpointURL, "/dav/files/", "/dav/uploads/", 1)
+}
+
+func (o *Object) getChunksUploadDir() (string, error) {
+	hasher := md5.New()
+	_, err := hasher.Write([]byte(o.filePath()))
+	if err != nil {
+		return "", fmt.Errorf("chunked upload couldn't hash URL: %w", err)
+	}
+	uploadDir := "rclone-chunked-upload-" + hex.EncodeToString(hasher.Sum(nil))
+	return uploadDir, nil
+}
+
+func (f *Fs) verifyChunkConfig() error {
+	if f.opt.ChunkSize != 0 && !validateNextCloudChunkedURL.MatchString(f.endpointURL) {
+		return errors.New("chunked upload with nextcloud must use /dav/files/USER endpoint not /webdav")
+	}
+
+	return nil
+}
+
+func (o *Object) shouldUseChunkedUpload(src fs.ObjectInfo) bool {
+	return o.fs.canChunk && o.fs.opt.ChunkSize > 0 && src.Size() > int64(o.fs.opt.ChunkSize)
+}
+
+func (o *Object) updateChunked(ctx context.Context, in0 io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	var uploadDir string
+
+	// see https://docs.nextcloud.com/server/24/developer_manual/client_apis/WebDAV/chunking.html#starting-a-chunked-upload
+	uploadDir, err = o.createChunksUploadDirectory(ctx)
+	if err != nil {
+		return err
+	}
+
+	partObj := &Object{
+		fs: o.fs,
+	}
+
+	// see https://docs.nextcloud.com/server/24/developer_manual/client_apis/WebDAV/chunking.html#uploading-chunks
+	err = o.uploadChunks(ctx, in0, src.Size(), partObj, uploadDir, options)
+	if err != nil {
+		return err
+	}
+
+	// see https://docs.nextcloud.com/server/24/developer_manual/client_apis/WebDAV/chunking.html#assembling-the-chunks
+	err = o.mergeChunks(ctx, uploadDir, options, src)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (o *Object) uploadChunks(ctx context.Context, in0 io.Reader, size int64, partObj *Object, uploadDir string, options []fs.OpenOption) error {
+	chunkSize := int64(partObj.fs.opt.ChunkSize)
+
+	// TODO: upload chunks in parallel for faster transfer speeds
+	for offset := int64(0); offset < size; offset += chunkSize {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		contentLength := chunkSize
+
+		// Last chunk may be smaller
+		if size-offset < contentLength {
+			contentLength = size - offset
+		}
+
+		endOffset := offset + contentLength - 1
+
+		partObj.remote = fmt.Sprintf("%s/%015d-%015d", uploadDir, offset, endOffset)
+		// Enable low-level HTTP 2 retries.
+		// 2022-04-28 15:59:06 ERROR : stuff/video.avi: Failed to copy: uploading chunk failed: Put "https://censored.com/remote.php/dav/uploads/Admin/rclone-chunked-upload-censored/000006113198080-000006123683840": http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error
+
+		buf := make([]byte, chunkSize)
+		in := readers.NewRepeatableLimitReaderBuffer(in0, buf, chunkSize)
+
+		getBody := func() (io.ReadCloser, error) {
+			// RepeatableReader{} plays well with accounting so rewinding doesn't make the progress buggy
+			if _, err := in.Seek(0, io.SeekStart); err == nil {
+				return nil, err
+			}
+
+			return io.NopCloser(in), nil
+		}
+
+		err := partObj.updateSimple(ctx, in, getBody, partObj.remote, contentLength, "application/x-www-form-urlencoded", nil, o.fs.chunksUploadURL, options...)
+		if err != nil {
+			return fmt.Errorf("uploading chunk failed: %w", err)
+		}
+	}
+	return nil
+}
+
+func (o *Object) createChunksUploadDirectory(ctx context.Context) (string, error) {
+	uploadDir, err := o.getChunksUploadDir()
+	if err != nil {
+		return uploadDir, err
+	}
+
+	err = o.purgeUploadedChunks(ctx, uploadDir)
+	if err != nil {
+		return "", fmt.Errorf("chunked upload couldn't purge upload directory: %w", err)
+	}
+
+	opts := rest.Opts{
+		Method:     "MKCOL",
+		Path:       uploadDir + "/",
+		NoResponse: true,
+		RootURL:    o.fs.chunksUploadURL,
+	}
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return "", fmt.Errorf("making upload directory failed: %w", err)
+	}
+	return uploadDir, err
+}
+
+func (o *Object) mergeChunks(ctx context.Context, uploadDir string, options []fs.OpenOption, src fs.ObjectInfo) error {
+	var resp *http.Response
+
+	// see https://docs.nextcloud.com/server/24/developer_manual/client_apis/WebDAV/chunking.html?highlight=chunk#assembling-the-chunks
+	opts := rest.Opts{
+		Method:     "MOVE",
+		Path:       path.Join(uploadDir, ".file"),
+		NoResponse: true,
+		Options:    options,
+		RootURL:    o.fs.chunksUploadURL,
+	}
+	destinationURL, err := rest.URLJoin(o.fs.endpoint, o.filePath())
+	if err != nil {
+		return fmt.Errorf("finalize chunked upload couldn't join URL: %w", err)
+	}
+	opts.ExtraHeaders = o.extraHeaders(ctx, src)
+	opts.ExtraHeaders["Destination"] = destinationURL.String()
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetryChunkMerge(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("finalize chunked upload failed, destinationURL: \"%s\": %w", destinationURL, err)
+	}
+	return err
+}
+
+func (o *Object) purgeUploadedChunks(ctx context.Context, uploadDir string) error {
+	// clean the upload directory if it exists (this means that a previous try didn't clean up properly).
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       uploadDir + "/",
+		NoResponse: true,
+		RootURL:    o.fs.chunksUploadURL,
+	}
+
+	err := o.fs.pacer.Call(func() (bool, error) {
+		resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
+
+		// directory doesn't exist, no need to purge
+		if resp.StatusCode == http.StatusNotFound {
+			return false, nil
+		}
+
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+
+	return err
+}

backend/webdav/webdav.go

@@ -19,6 +19,7 @@ import (
 	"net/url"
 	"os/exec"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -42,7 +43,7 @@ import (
 )
 
 const (
-	minSleep      = 10 * time.Millisecond
+	minSleep      = fs.Duration(10 * time.Millisecond)
 	maxSleep      = 2 * time.Second
 	decayConstant = 2   // bigger for slower decay, exponential
 	defaultDepth  = "1" // depth for PROPFIND
@@ -76,6 +77,9 @@ func init() {
 			Name: "vendor",
 			Help: "Name of the WebDAV site/service/software you are using.",
 			Examples: []fs.OptionExample{{
+				Value: "fastmail",
+				Help:  "Fastmail Files",
+			}, {
 				Value: "nextcloud",
 				Help:  "Nextcloud",
 			}, {
@@ -124,6 +128,22 @@ You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'
 `,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
+		}, {
+			Name:     "pacer_min_sleep",
+			Help:     "Minimum time to sleep between API calls.",
+			Default:  minSleep,
+			Advanced: true,
+		}, {
+			Name: "nextcloud_chunk_size",
+			Help: `Nextcloud upload chunk size.
+
+We recommend configuring your NextCloud instance to increase the max chunk size to 1 GB for better upload performances.
+See https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/big_file_upload_configuration.html#adjust-chunk-size-on-nextcloud-side
+
+Set to 0 to disable chunked uploading.
+`,
+			Advanced: true,
+			Default:  10 * fs.Mebi, // Default NextCloud `max_chunk_size` is `10 MiB`. See https://github.com/nextcloud/server/blob/0447b53bda9fe95ea0cbed765aa332584605d652/apps/files/lib/App.php#L57
 		}},
 	})
 }
@@ -138,6 +158,8 @@ type Options struct {
 	BearerTokenCommand string               `config:"bearer_token_command"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 	Headers            fs.CommaSepList      `config:"headers"`
+	PacerMinSleep      fs.Duration          `config:"pacer_min_sleep"`
+	ChunkSize          fs.SizeSuffix        `config:"nextcloud_chunk_size"`
 }
 
 // Fs represents a remote webdav
@@ -155,9 +177,12 @@ type Fs struct {
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
 	checkBeforePurge   bool          // enables extra check that directory to purge really exists
-	hasMD5             bool          // set if can use owncloud style checksums for MD5
-	hasSHA1            bool          // set if can use owncloud style checksums for SHA1
+	hasOCMD5           bool          // set if can use owncloud style checksums for MD5
+	hasOCSHA1          bool          // set if can use owncloud style checksums for SHA1
+	hasMESHA1          bool          // set if can use fastmail style checksums for SHA1
 	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
+	chunksUploadURL    string        // upload URL for nextcloud chunked
+	canChunk           bool          // set if nextcloud and nextcloud_chunk_size is set
 }
 
 // Object describes a webdav object
@@ -278,7 +303,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
 		},
 		NoRedirect: true,
 	}
-	if f.hasMD5 || f.hasSHA1 {
+	if f.hasOCMD5 || f.hasOCSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -411,7 +436,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:         *opt,
 		endpoint:    u,
 		endpointURL: u.String(),
-		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}
 
@@ -543,19 +568,32 @@ func (f *Fs) fetchAndSetBearerToken() error {
 	return nil
 }
 
+var validateNextCloudChunkedURL = regexp.MustCompile(`^.*/dav/files/[^/]+/?$`)
+
 // setQuirks adjusts the Fs for the vendor passed in
 func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 	switch vendor {
+	case "fastmail":
+		f.canStream = true
+		f.precision = time.Second
+		f.useOCMtime = true
+		f.hasMESHA1 = true
 	case "owncloud":
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasMD5 = true
-		f.hasSHA1 = true
+		f.hasOCMD5 = true
+		f.hasOCSHA1 = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasSHA1 = true
+		f.hasOCSHA1 = true
+		f.canChunk = true
+		if err := f.verifyChunkConfig(); err != nil {
+			return err
+		}
+		f.chunksUploadURL = f.getChunksUploadURL()
+		fs.Logf(nil, "Chunks temporary upload directory: %s", f.chunksUploadURL)
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -667,7 +705,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 			"Depth": depth,
 		},
 	}
-	if f.hasMD5 || f.hasSHA1 {
+	if f.hasOCMD5 || f.hasOCSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -996,7 +1034,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 	dstPath := f.filePath(remote)
 	err := f.mkParentDir(ctx, dstPath)
 	if err != nil {
-		return nil, fmt.Errorf("Copy mkParentDir failed: %w", err)
+		return nil, fmt.Errorf("copy mkParentDir failed: %w", err)
 	}
 	destinationURL, err := rest.URLJoin(f.endpoint, dstPath)
 	if err != nil {
@@ -1021,11 +1059,11 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 		return srcFs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("Copy call failed: %w", err)
+		return nil, fmt.Errorf("copy call failed: %w", err)
 	}
 	dstObj, err := f.NewObject(ctx, remote)
 	if err != nil {
-		return nil, fmt.Errorf("Copy NewObject failed: %w", err)
+		return nil, fmt.Errorf("copy NewObject failed: %w", err)
 	}
 	return dstObj, nil
 }
@@ -1126,10 +1164,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	hashes := hash.Set(hash.None)
-	if f.hasMD5 {
+	if f.hasOCMD5 {
 		hashes.Add(hash.MD5)
 	}
-	if f.hasSHA1 {
+	if f.hasOCSHA1 || f.hasMESHA1 {
 		hashes.Add(hash.SHA1)
 	}
 	return hashes
@@ -1197,10 +1235,10 @@ func (o *Object) Remote() string {
 
 // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == hash.MD5 && o.fs.hasMD5 {
+	if t == hash.MD5 && o.fs.hasOCMD5 {
 		return o.md5, nil
 	}
-	if t == hash.SHA1 && o.fs.hasSHA1 {
+	if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) {
 		return o.sha1, nil
 	}
 	return "", hash.ErrUnsupported
@@ -1222,12 +1260,12 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
-	if o.fs.hasMD5 || o.fs.hasSHA1 {
+	if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
 		hashes := info.Hashes()
-		if o.fs.hasSHA1 {
+		if o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
 			o.sha1 = hashes[hash.SHA1]
 		}
-		if o.fs.hasMD5 {
+		if o.fs.hasOCMD5 {
 			o.md5 = hashes[hash.MD5]
 		}
 	}
@@ -1304,36 +1342,72 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update mkParentDir failed: %w", err)
 	}
 
-	size := src.Size()
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:        "PUT",
-		Path:          o.filePath(),
-		Body:          in,
-		NoResponse:    true,
-		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
-		ContentType:   fs.MimeType(ctx, src),
-		Options:       options,
+	if o.shouldUseChunkedUpload(src) {
+		fs.Debugf(src, "Update will use the chunked upload strategy")
+		err = o.updateChunked(ctx, in, src, options...)
+		if err != nil {
+			return err
+		}
+	} else {
+		fs.Debugf(src, "Update will use the normal upload strategy (no chunks)")
+		contentType := fs.MimeType(ctx, src)
+		filePath := o.filePath()
+		extraHeaders := o.extraHeaders(ctx, src)
+		// TODO: define getBody() to enable low-level HTTP/2 retries
+		err = o.updateSimple(ctx, in, nil, filePath, src.Size(), contentType, extraHeaders, o.fs.endpointURL, options...)
+		if err != nil {
+			return err
+		}
 	}
-	if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 {
-		opts.ExtraHeaders = map[string]string{}
+
+	// read metadata from remote
+	o.hasMetaData = false
+	return o.readMetaData(ctx)
+}
+
+func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string {
+	extraHeaders := map[string]string{}
+	if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
 		if o.fs.useOCMtime {
-			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
+			extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
 		}
 		// Set one upload checksum
 		// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
 		// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
-		if o.fs.hasSHA1 {
+		if o.fs.hasOCSHA1 {
 			if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
+				extraHeaders["OC-Checksum"] = "SHA1:" + sha1
 			}
 		}
-		if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
+		if o.fs.hasOCMD5 && extraHeaders["OC-Checksum"] == "" {
 			if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
+				extraHeaders["OC-Checksum"] = "MD5:" + md5
 			}
 		}
 	}
+	return extraHeaders
+}
+
+// Standard update in one request (no chunks)
+func (o *Object) updateSimple(ctx context.Context, body io.Reader, getBody func() (io.ReadCloser, error), filePath string, size int64, contentType string, extraHeaders map[string]string, rootURL string, options ...fs.OpenOption) (err error) {
+	var resp *http.Response
+
+	if extraHeaders == nil {
+		extraHeaders = map[string]string{}
+	}
+
+	opts := rest.Opts{
+		Method:        "PUT",
+		Path:          filePath,
+		GetBody:       getBody,
+		Body:          body,
+		NoResponse:    true,
+		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
+		ContentType:   contentType,
+		Options:       options,
+		ExtraHeaders:  extraHeaders,
+		RootURL:       rootURL,
+	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)
 		return o.fs.shouldRetry(ctx, resp, err)
@@ -1349,9 +1423,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		_ = o.Remove(ctx)
 		return err
 	}
-	// read metadata from remote
-	o.hasMetaData = false
-	return o.readMetaData(ctx)
+	return nil
+
 }
 
 // Remove an object

backend/webdav/webdav_test.go

@@ -1,10 +1,10 @@
 // Test Webdav filesystem interface
-package webdav_test
+package webdav
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/webdav"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
@@ -13,7 +13,10 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestWebdavNextcloud:",
-		NilObject:  (*webdav.Object)(nil),
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: 1 * fs.Mebi,
+		},
 	})
 }
 
@@ -24,7 +27,10 @@ func TestIntegration2(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestWebdavOwncloud:",
-		NilObject:  (*webdav.Object)(nil),
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			Skip: true,
+		},
 	})
 }
 
@@ -35,7 +41,10 @@ func TestIntegration3(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestWebdavRclone:",
-		NilObject:  (*webdav.Object)(nil),
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			Skip: true,
+		},
 	})
 }
 
@@ -46,6 +55,10 @@ func TestIntegration4(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestWebdavNTLM:",
-		NilObject:  (*webdav.Object)(nil),
+		NilObject:  (*Object)(nil),
 	})
 }
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadChunkSize(cs)
+}

backend/zoho/zoho.go

@@ -331,15 +331,6 @@ func parsePath(path string) (root string) {
 	return
 }
 
-func (f *Fs) splitPath(remote string) (directory, leaf string) {
-	directory, leaf = dircache.SplitPath(remote)
-	if f.root != "" {
-		// Adds the root folder to the path to get a full path
-		directory = path.Join(f.root, directory)
-	}
-	return
-}
-
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)

bin/cross-compile.go

@@ -225,7 +225,7 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
 		"StringFileInfo": M{
 			"CompanyName":      "https://rclone.org",
 			"ProductName":      "Rclone",
-			"FileDescription":  "Rsync for cloud storage",
+			"FileDescription":  "Rclone",
 			"InternalName":     "rclone",
 			"OriginalFilename": "rclone.exe",
 			"LegalCopyright":   "The Rclone Authors",

bin/make_manual.py

@@ -64,6 +64,7 @@ docs = [
     "sia.md",
     "swift.md",
     "pcloud.md",
+    "pikpak.md",
     "premiumizeme.md",
     "putio.md",
     "seafile.md",

bin/upload-github

@@ -26,7 +26,8 @@ echo "Making release ${VERSION} anchor ${ANCHOR} to repo ${REPO}"
 gh release create "${VERSION}" \
     --repo ${REPO} \
     --title "rclone ${VERSION}" \
-    --notes-file "/tmp/${VERSION}-release-notes"
+    --notes-file "/tmp/${VERSION}-release-notes" \
+    --draft=true
 
 for build in build/*; do
     case $build in
@@ -40,6 +41,10 @@ for build in build/*; do
         "${build}"
 done
 
+gh release edit "${VERSION}" \
+    --repo ${REPO} \
+    --draft=false
+
 gh release view "${VERSION}" \
     --repo ${REPO}
 

cmd/bisync/cmd.go

@@ -128,7 +128,6 @@ var commandDefinition = &cobra.Command{
 		ctx := context.Background()
 		opt := Opt
 		opt.applyContext(ctx)
-
 		if tzLocal {
 			TZ = time.Local
 		}

cmd/bisync/rc.go

@@ -34,7 +34,7 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 		if maxDelete < 0 || maxDelete > 100 {
 			return nil, rc.NewErrParamInvalid(errors.New("maxDelete must be a percentage between 0 and 100"))
 		}
-		ci.MaxDelete = maxDelete
+		opt.MaxDelete = int(maxDelete)
 	} else if rc.NotErrParamNotFound(err) {
 		return nil, err
 	}

cmd/bisync/testdata/test_check_access_filters/scenario.txt

@@ -3,8 +3,8 @@ test check-access-filters
 # NOTE: Include Other tests may result in listing diffs due to rclone processing order change. False fail.
 #
 # Tests are done in two phases:
-#  - EXCLUDE OTHER tests check that RCLONE_TEST files are only found in the explicity included directories
-#  - INCLUDE OTHER tesss check that RCLONE_TEST files are found in all directories not explicity excluded
+#  - EXCLUDE OTHER tests check that RCLONE_TEST files are only found in the explicitly included directories
+#  - INCLUDE OTHER tesss check that RCLONE_TEST files are found in all directories not explicitly excluded
 #
 # Each phase checks that:
 #  - missing RCLONE_TEST files in don't care directories don't cause failures

cmd/check/check.go

@@ -72,6 +72,9 @@ you what happened to it. These are reminiscent of diff files.
 - |+ path| means path was missing on the destination, so only in the source
 - |* path| means path was present in source and destination but different.
 - |! path| means there was an error reading or hashing the source or dest.
+
+The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
+option for more information.
 `, "|", "`")
 
 // GetCheckOpt gets the options corresponding to the check flags
@@ -142,7 +145,7 @@ match.  It doesn't alter the source or destination.
 
 For the [crypt](/crypt/) remote there is a dedicated command,
 [cryptcheck](/commands/rclone_cryptcheck/), that are able to check
-the checksums of the crypted files.
+the checksums of the encrypted files.
 
 If you supply the |--size-only| flag, it will only compare the sizes not
 the hashes as well.  Use this for a quick check.

cmd/cmount/mount.go

@@ -160,7 +160,11 @@ func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error,
 	fsys := NewFS(VFS)
 	host := fuse.NewFileSystemHost(fsys)
 	host.SetCapReaddirPlus(true) // only works on Windows
-	host.SetCapCaseInsensitive(f.Features().CaseInsensitive)
+	if opt.CaseInsensitive.Valid {
+		host.SetCapCaseInsensitive(opt.CaseInsensitive.Value)
+	} else {
+		host.SetCapCaseInsensitive(f.Features().CaseInsensitive)
+	}
 
 	// Create options
 	options := mountOptions(VFS, opt.DeviceName, mountpoint, opt)

cmd/cmount/mount_brew.go

@@ -28,7 +28,7 @@ func init() {
 // returns an error, and an error channel for the serve process to
 // report an error when fusermount is called.
 func mount(_ *vfs.VFS, _ string, _ *mountlib.Options) (<-chan error, func() error, error) {
-	return nil, nil, errors.New("mount is not supported on MacOS when installed via Homebrew. " +
-		"Please install the binaries available at https://rclone." +
-		"org/downloads/ instead if you want to use the mount command")
+	return nil, nil, errors.New("rclone mount is not supported on MacOS when rclone is installed via Homebrew. " +
+		"Please install the rclone binaries available at https://rclone.org/downloads/ " +
+		"instead if you want to use the rclone mount command")
 }

cmd/cryptcheck/cryptcheck.go

@@ -22,11 +22,11 @@ func init() {
 
 var commandDefinition = &cobra.Command{
 	Use:   "cryptcheck remote:path cryptedremote:path",
-	Short: `Cryptcheck checks the integrity of a crypted remote.`,
+	Short: `Cryptcheck checks the integrity of an encrypted remote.`,
 	Long: `
 rclone cryptcheck checks a remote against a [crypted](/crypt/) remote.
 This is the equivalent of running rclone [check](/commands/rclone_check/),
-but able to check the checksums of the crypted remote.
+but able to check the checksums of the encrypted remote.
 
 For it to work the underlying remote of the cryptedremote must support
 some kind of checksum.
@@ -59,7 +59,7 @@ After it has run it will log the status of the encryptedremote:.
 	},
 }
 
-// cryptCheck checks the integrity of a crypted remote
+// cryptCheck checks the integrity of an encrypted remote
 func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
 	// Check to see fcrypt is a crypt
 	fcrypt, ok := fdst.(*crypt.Fs)

cmd/mount/dir.go

@@ -250,7 +250,7 @@ func (d *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (node fusefs.No
 	defer log.Trace(d, "name=%v, mode=%d, rdev=%d", req.Name, req.Mode, req.Rdev)("node=%v, err=%v", &node, &err)
 	if req.Rdev != 0 {
 		fs.Errorf(d, "Can't create device node %q", req.Name)
-		return nil, fuse.EIO
+		return nil, fuse.Errno(syscall.EIO)
 	}
 	var cReq = fuse.CreateRequest{
 		Name:  req.Name,

cmd/mount/fs.go

@@ -82,11 +82,11 @@ func translateError(err error) error {
 	case vfs.OK:
 		return nil
 	case vfs.ENOENT, fs.ErrorDirNotFound, fs.ErrorObjectNotFound:
-		return fuse.ENOENT
+		return fuse.Errno(syscall.ENOENT)
 	case vfs.EEXIST, fs.ErrorDirExists:
-		return fuse.EEXIST
+		return fuse.Errno(syscall.EEXIST)
 	case vfs.EPERM, fs.ErrorPermissionDenied:
-		return fuse.EPERM
+		return fuse.Errno(syscall.EPERM)
 	case vfs.ECLOSED:
 		return fuse.Errno(syscall.EBADF)
 	case vfs.ENOTEMPTY:

cmd/mountlib/help.go

@@ -254,6 +254,44 @@ example above.
 Note that mapping to a directory path, instead of a drive letter,
 does not suffer from the same limitations.
 
+### Mounting on macOS
+
+Mounting on macOS can be done either via [macFUSE](https://osxfuse.github.io/) 
+(also known as osxfuse) or [FUSE-T](https://www.fuse-t.org/). macFUSE is a traditional
+FUSE driver utilizing a macOS kernel extension (kext). FUSE-T is an alternative FUSE system
+which "mounts" via an NFSv4 local server.
+
+#### FUSE-T Limitations, Caveats, and Notes
+
+There are some limitations, caveats, and notes about how it works. These are current as 
+of FUSE-T version 1.0.14.
+
+##### ModTime update on read
+
+As per the [FUSE-T wiki](https://github.com/macos-fuse-t/fuse-t/wiki#caveats):
+
+> File access and modification times cannot be set separately as it seems to be an 
+> issue with the NFS client which always modifies both. Can be reproduced with 
+> 'touch -m' and 'touch -a' commands
+
+This means that viewing files with various tools, notably macOS Finder, will cause rlcone
+to update the modification time of the file. This may make rclone upload a full new copy
+of the file.
+
+##### Unicode Normalization
+
+Rclone includes flags for unicode normalization with macFUSE that should be updated
+for FUSE-T. See [this forum post](https://forum.rclone.org/t/some-unicode-forms-break-mount-on-macos-with-fuse-t/36403)
+and [FUSE-T issue #16](https://github.com/macos-fuse-t/fuse-t/issues/16). The following
+flag should be added to the |rclone mount| command.
+
+    -o modules=iconv,from_code=UTF-8,to_code=UTF-8
+    
+##### Read Only mounts
+
+When mounting with |--read-only|, attempts to write to files will fail *silently* as
+opposed to with a clear warning as in macFUSE.
+
 ### Limitations
 
 Without the use of |--vfs-cache-mode| this can only write files

cmd/mountlib/mount.go

@@ -48,6 +48,7 @@ type Options struct {
 	DaemonTimeout      time.Duration // OSXFUSE only
 	AsyncRead          bool
 	NetworkMode        bool // Windows only
+	CaseInsensitive    fs.Tristate
 }
 
 // DefaultOpt is the default values for creating the mount
@@ -139,6 +140,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &Opt.MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads (not supported on Windows)")
 	flags.BoolVarP(flagSet, &Opt.WritebackCache, "write-back-cache", "", Opt.WritebackCache, "Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows)")
 	flags.StringVarP(flagSet, &Opt.DeviceName, "devname", "", Opt.DeviceName, "Set the device name - default is remote:path")
+	flags.FVarP(flagSet, &Opt.CaseInsensitive, "mount-case-insensitive", "", "Tell the OS the mount is case insensitive (true) or sensitive (false) regardless of the backend (auto)")
 	// Windows and OSX
 	flags.StringVarP(flagSet, &Opt.VolumeName, "volname", "", Opt.VolumeName, "Set the volume name (supported on Windows and OSX only)")
 	// OSX only

cmd/mountlib/rc.go

@@ -98,7 +98,7 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	}
 
 	if mountOpt.Daemon {
-		return nil, errors.New("Daemon Option not supported over the API")
+		return nil, errors.New("daemon option not supported over the API")
 	}
 
 	mountType, err := in.GetString("mountType")
@@ -111,7 +111,7 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	}
 	mountType, mountFn := ResolveMountMethod(mountType)
 	if mountFn == nil {
-		return nil, errors.New("Mount Option specified is not registered, or is invalid")
+		return nil, errors.New("mount option specified is not registered, or is invalid")
 	}
 
 	// Get Fs.fs to be mounted from fs parameter in the params

cmd/ncdu/ncdu.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/cmd/ncdu/scan"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fspath"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rivo/uniseg"
 	"github.com/spf13/cobra"
@@ -81,7 +82,7 @@ the remote you can also use the [size](/commands/rclone_size/) command.
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
-			return NewUI(fsrc).Show()
+			return NewUI(fsrc).Run()
 		})
 	},
 }
@@ -353,7 +354,7 @@ func (u *UI) hasEmptyDir() bool {
 }
 
 // Draw the current screen
-func (u *UI) Draw() error {
+func (u *UI) Draw() {
 	ctx := context.Background()
 	w, h := u.s.Size()
 	u.dirListHeight = h - 3
@@ -489,8 +490,6 @@ func (u *UI) Draw() error {
 	if u.showBox {
 		u.Box()
 	}
-	u.s.Show()
-	return nil
 }
 
 // Move the cursor this many spaces adjusting the viewport as necessary
@@ -900,8 +899,8 @@ func NewUI(f fs.Fs) *UI {
 	}
 }
 
-// Show shows the user interface
-func (u *UI) Show() error {
+// Run shows the user interface
+func (u *UI) Run() error {
 	var err error
 	u.s, err = tcell.NewScreen()
 	if err != nil {
@@ -911,6 +910,28 @@ func (u *UI) Show() error {
 	if err != nil {
 		return fmt.Errorf("screen init: %w", err)
 	}
+
+	// Hijack fs.LogPrint so that it doesn't corrupt the screen.
+	if logPrint := fs.LogPrint; !log.Redirected() {
+		type log struct {
+			text  string
+			level fs.LogLevel
+		}
+		var logs []log
+		fs.LogPrint = func(level fs.LogLevel, text string) {
+			if len(logs) > 100 {
+				logs = logs[len(logs)-100:]
+			}
+			logs = append(logs, log{level: level, text: text})
+		}
+		defer func() {
+			fs.LogPrint = logPrint
+			for i := range logs {
+				logPrint(logs[i].level, logs[i].text)
+			}
+		}()
+	}
+
 	defer u.s.Fini()
 
 	// scan the disk in the background
@@ -924,10 +945,6 @@ func (u *UI) Show() error {
 	// Main loop, waiting for events and channels
 outer:
 	for {
-		err := u.Draw()
-		if err != nil {
-			return fmt.Errorf("draw failed: %w", err)
-		}
 		select {
 		case root := <-rootChan:
 			u.root = root
@@ -938,16 +955,14 @@ outer:
 			}
 			u.listing = false
 		case <-updated:
-			// redraw
 			// TODO: might want to limit updates per second
 			u.sortCurrentDir()
 		case ev := <-events:
 			switch ev := ev.(type) {
 			case *tcell.EventResize:
-				if u.root != nil {
-					u.sortCurrentDir() // redraw
-				}
+				u.Draw()
 				u.s.Sync()
+				continue // don't draw again
 			case *tcell.EventKey:
 				var c rune
 				if k := ev.Key(); k == tcell.KeyRune {
@@ -1026,11 +1041,15 @@ outer:
 				// Refresh the screen. Not obvious what key to map
 				// this onto, but ^L is a common choice.
 				case key(tcell.KeyCtrlL):
+					u.Draw()
 					u.s.Sync()
+					continue // don't draw again
 				}
 			}
 		}
-		// listen to key presses, etc.
+
+		u.Draw()
+		u.s.Show()
 	}
 	return nil
 }

cmd/progress.go

@@ -75,14 +75,13 @@ func startProgress() func() {
 
 // state for the progress printing
 var (
-	nlines     = 0 // number of lines in the previous stats block
-	progressMu sync.Mutex
+	nlines = 0 // number of lines in the previous stats block
 )
 
 // printProgress prints the progress with an optional log
 func printProgress(logMessage string) {
-	progressMu.Lock()
-	defer progressMu.Unlock()
+	operations.StdoutMutex.Lock()
+	defer operations.StdoutMutex.Unlock()
 
 	var buf bytes.Buffer
 	w, _ := terminal.GetSize()

cmd/rcat/rcat.go

@@ -52,10 +52,11 @@ and actually stream it, even if remote backend doesn't support streaming.
 size of the stream is different in length to the ` + "`--size`" + ` passed in
 then the transfer will likely fail.
 
-Note that the upload can also not be retried because the data is
-not kept around until the upload succeeds. If you need to transfer
-a lot of data, you're better off caching locally and then
-` + "`rclone move`" + ` it to the destination.`,
+Note that the upload cannot be retried because the data is not stored.
+If the backend supports multipart uploading then individual chunks can
+be retried. If you need to transfer a lot of data, you may be better
+off caching it locally and then ` + "`rclone move`" + ` it to the
+destination which can use retries.`,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.38",
 	},

cmd/rcd/rcd.go

@@ -32,7 +32,7 @@ for GET requests on the URL passed in.  It will also open the URL in
 the browser when rclone is run.
 
 See the [rc documentation](/rc/) for more info on the rc flags.
-` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp,
+` + libhttp.Help(rcflags.FlagPrefix) + libhttp.TemplateHelp(rcflags.FlagPrefix) + libhttp.AuthHelp(rcflags.FlagPrefix),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.45",
 	},

cmd/serve/ftp/ftp.go

@@ -469,7 +469,7 @@ func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64,
 	}
 	defer closeIO(path, of)
 
-	_, err = of.Seek(0, os.SEEK_END)
+	_, err = of.Seek(0, io.SeekEnd)
 	if err != nil {
 		return 0, err
 	}

cmd/serve/http/http.go

@@ -44,11 +44,15 @@ var DefaultOpt = Options{
 // Opt is options set by command line flags
 var Opt = DefaultOpt
 
+// flagPrefix is the prefix used to uniquely identify command line flags.
+// It is intentionally empty for this package.
+const flagPrefix = ""
+
 func init() {
 	flagSet := Command.Flags()
-	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
-	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
-	libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
+	libhttp.AddAuthFlagsPrefix(flagSet, flagPrefix, &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP)
+	libhttp.AddTemplateFlagsPrefix(flagSet, flagPrefix, &Opt.Template)
 	vfsflags.AddFlags(flagSet)
 	proxyflags.AddFlags(flagSet)
 }
@@ -68,7 +72,7 @@ The server will log errors.  Use ` + "`-v`" + ` to see access logs.
 
 ` + "`--bwlimit`" + ` will be respected for file transfers.  Use ` + "`--stats`" + ` to
 control the stats printing.
-` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help,
+` + libhttp.Help(flagPrefix) + libhttp.TemplateHelp(flagPrefix) + libhttp.AuthHelp(flagPrefix) + vfs.Help + proxy.Help,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},

cmd/serve/restic/restic.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"time"
 
+	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
+
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/rclone/rclone/cmd"
@@ -47,10 +49,14 @@ var DefaultOpt = Options{
 // Opt is options set by command line flags
 var Opt = DefaultOpt
 
+// flagPrefix is the prefix used to uniquely identify command line flags.
+// It is intentionally empty for this package.
+const flagPrefix = ""
+
 func init() {
 	flagSet := Command.Flags()
-	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
-	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
+	libhttp.AddAuthFlagsPrefix(flagSet, flagPrefix, &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP)
 	flags.BoolVarP(flagSet, &Opt.Stdio, "stdio", "", false, "Run an HTTP2 server on stdin/stdout")
 	flags.BoolVarP(flagSet, &Opt.AppendOnly, "append-only", "", false, "Disallow deletion of repository data")
 	flags.BoolVarP(flagSet, &Opt.PrivateRepos, "private-repos", "", false, "Users can only access their private repo")
@@ -142,7 +148,7 @@ these **must** end with /.  Eg
 
 The` + "`--private-repos`" + ` flag can be used to limit users to repositories starting
 with a path of ` + "`/<username>/`" + `.
-` + libhttp.Help + libhttp.AuthHelp,
+` + libhttp.Help(flagPrefix) + libhttp.AuthHelp(flagPrefix),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.40",
 	},
@@ -173,7 +179,17 @@ with a path of ` + "`/<username>/`" + `.
 				return nil
 			}
 			fs.Logf(s.f, "Serving restic REST API on %s", s.URLs())
+
+			if err := sysdnotify.Ready(); err != nil {
+				fs.Logf(s.f, "failed to notify ready to systemd: %v", err)
+			}
+
 			s.Wait()
+
+			if err := sysdnotify.Stopping(); err != nil {
+				fs.Logf(s.f, "failed to notify stopping to systemd: %v", err)
+			}
+
 			return nil
 		})
 	},

cmd/serve/sftp/sftp.go

@@ -108,7 +108,7 @@ which can lead to "corrupted on transfer" errors. This is the case because
 the client chooses indiscriminately which server to send commands to while
 the servers all have different views of the state of the filing system.
 
-The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing
+The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from being
 used. Omitting "restrict" and using  ` + "`--sftp-path-override`" + ` to enable
 checksumming is possible but less secure and you could use the SFTP server
 provided by OpenSSH in this case.

cmd/serve/webdav/webdav.go

@@ -48,10 +48,14 @@ var DefaultOpt = Options{
 // Opt is options set by command line flags
 var Opt = DefaultOpt
 
+// flagPrefix is the prefix used to uniquely identify command line flags.
+// It is intentionally empty for this package.
+const flagPrefix = ""
+
 func init() {
 	flagSet := Command.Flags()
-	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
-	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
+	libhttp.AddAuthFlagsPrefix(flagSet, flagPrefix, &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP)
 	libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
 	vfsflags.AddFlags(flagSet)
 	proxyflags.AddFlags(flagSet)
@@ -103,7 +107,7 @@ Create a new DWORD BasicAuthLevel with value 2.
 
 https://learn.microsoft.com/en-us/office/troubleshoot/powerpoint/office-opens-blank-from-sharepoint
 
-` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help,
+` + libhttp.Help(flagPrefix) + libhttp.TemplateHelp(flagPrefix) + libhttp.AuthHelp(flagPrefix) + vfs.Help + proxy.Help,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},

cmd/size/size.go

@@ -39,7 +39,7 @@ recursion.
 
 Some backends do not always provide file sizes, see for example
 [Google Photos](/googlephotos/#size) and
-[Google Drive](/drive/#limitations-of-google-docs).
+[Google Docs](/drive/#limitations-of-google-docs).
 Rclone will then show a notice in the log indicating how many such
 files were encountered, and count them in as empty files in the output
 of the size command.

cmd/test/test.go

@@ -16,7 +16,7 @@ var Command = &cobra.Command{
 	Short: `Run a test command`,
 	Long: `Rclone test is used to run test commands.
 
-Select which test comand you want with the subcommand, eg
+Select which test command you want with the subcommand, eg
 
     rclone test memory remote:
 

cmd/touch/touch.go

@@ -141,7 +141,7 @@ func Touch(ctx context.Context, f fs.Fs, remote string) error {
 	file, err := f.NewObject(ctx, remote)
 	if err != nil {
 		if errors.Is(err, fs.ErrorObjectNotFound) {
-			// Touching non-existant path, possibly creating it as new file
+			// Touching non-existent path, possibly creating it as new file
 			if remote == "" {
 				fs.Logf(f, "Not touching empty directory")
 				return nil

cmd/tree/tree.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs/dirtree"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/terminal"
 	"github.com/spf13/cobra"
 )
@@ -27,6 +28,7 @@ var (
 	outFileName string
 	noReport    bool
 	sort        string
+	enc         = encoder.OS
 )
 
 func init() {
@@ -163,7 +165,7 @@ type FileInfo struct {
 
 // Name is base name of the file
 func (to *FileInfo) Name() string {
-	return path.Base(to.entry.Remote())
+	return enc.FromStandardName(path.Base(to.entry.Remote()))
 }
 
 // Size in bytes for regular files; system-dependent for others
@@ -197,7 +199,7 @@ func (to *FileInfo) Sys() interface{} {
 
 // String returns the full path
 func (to *FileInfo) String() string {
-	return to.entry.Remote()
+	return filepath.FromSlash(enc.FromStandardPath(to.entry.Remote()))
 }
 
 // Fs maps an fs.Fs into a tree.Fs
@@ -212,6 +214,7 @@ func NewFs(dirs dirtree.DirTree) Fs {
 func (dirs Fs) Stat(filePath string) (fi os.FileInfo, err error) {
 	defer log.Trace(nil, "filePath=%q", filePath)("fi=%+v, err=%v", &fi, &err)
 	filePath = filepath.ToSlash(filePath)
+	filePath = enc.ToStandardPath(filePath)
 	filePath = strings.TrimLeft(filePath, "/")
 	if filePath == "" {
 		return &FileInfo{fs.NewDir("", time.Now())}, nil
@@ -227,13 +230,14 @@ func (dirs Fs) Stat(filePath string) (fi os.FileInfo, err error) {
 func (dirs Fs) ReadDir(dir string) (names []string, err error) {
 	defer log.Trace(nil, "dir=%s", dir)("names=%+v, err=%v", &names, &err)
 	dir = filepath.ToSlash(dir)
+	dir = enc.ToStandardPath(dir)
 	dir = strings.TrimLeft(dir, "/")
 	entries, ok := dirs[dir]
 	if !ok {
 		return nil, fmt.Errorf("couldn't find directory %q", dir)
 	}
 	for _, entry := range entries {
-		names = append(names, path.Base(entry.Remote()))
+		names = append(names, enc.FromStandardName(path.Base(entry.Remote())))
 	}
 	return
 }

cmdtest/cmdtest_test.go

@@ -43,7 +43,7 @@ const rcloneTestMain = "RCLONE_TEST_MAIN"
 
 // rcloneExecMain calls rclone with the given environment and arguments.
 // The environment variables are in a single string separated by ;
-// The terminal output is retuned as a string.
+// The terminal output is returned as a string.
 func rcloneExecMain(env string, args ...string) (string, error) {
 	_, found := os.LookupEnv(rcloneTestMain)
 	if !found {
@@ -62,7 +62,7 @@ func rcloneExecMain(env string, args ...string) (string, error) {
 // rcloneEnv calls rclone with the given environment and arguments.
 // The environment variables are in a single string separated by ;
 // The test config file is automatically configured in RCLONE_CONFIG.
-// The terminal output is retuned as a string.
+// The terminal output is returned as a string.
 func rcloneEnv(env string, args ...string) (string, error) {
 	envConfig := env
 	if testConfig != "" {
@@ -76,7 +76,7 @@ func rcloneEnv(env string, args ...string) (string, error) {
 
 // rclone calls rclone with the given arguments, E.g. "version","--help".
 // The test config file is automatically configured in RCLONE_CONFIG.
-// The terminal output is retuned as a string.
+// The terminal output is returned as a string.
 func rclone(args ...string) (string, error) {
 	return rcloneEnv("", args...)
 }

contrib/docker-plugin/managed/Dockerfile

@@ -8,7 +8,7 @@ FROM alpine:latest
 COPY --from=binaries /usr/local/bin/rclone /usr/bin/rclone
 
 RUN mkdir -p /data/config /data/cache /mnt \
- && apk --no-cache add ca-certificates fuse tzdata \
+ && apk --no-cache add ca-certificates fuse3 tzdata \
  && echo "user_allow_other" >> /etc/fuse.conf \
  && rclone version
 

docs/content/_index.md

@@ -121,6 +121,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
 {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
 {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
+{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
 {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
 {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
 {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}
@@ -132,7 +133,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
 {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
-{{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}}
+{{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
 {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
 {{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-object-storage" >}}
@@ -151,6 +152,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Oracle Object Storage" home="https://www.oracle.com/cloud/storage/object-storage" config="/oracleobjectstorage/" >}}
 {{< provider name="ownCloud" home="https://owncloud.org/" config="/webdav/#owncloud" >}}
 {{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}}
+{{< provider name="PikPak" home="https://mypikpak.com/" config="/pikpak/" >}}
 {{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}}
 {{< provider name="put.io" home="https://put.io/" config="/putio/" >}}
 {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}

docs/content/authors.md

@@ -13,7 +13,7 @@ Authors
 Contributors
 ------------
 
-{{< rem `email addresses removed from here need to be addeed to
+{{< rem `email addresses removed from here need to be added to
 bin/.ignore-emails to make sure update-authors.py doesn't immediately
 put them back in again.` >}}
 
@@ -597,6 +597,7 @@ put them back in again.` >}}
   * Christian Galo <36752715+cgalo5758@users.noreply.github.com>
   * Erik van Velzen <erik@evanv.nl>
   * Derek Battams <derek@battams.ca>
+  * Paul <devnoname120@gmail.com>
   * SimonLiu <simonliu009@users.noreply.github.com>
   * Hugo Laloge <hla@lescompanions.com>
   * Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com>
@@ -689,3 +690,26 @@ put them back in again.` >}}
   * Gerard Bosch <30733556+gerardbosch@users.noreply.github.com>
   * ToBeFree <github@tfrei.de>
   * NodudeWasTaken <75137537+NodudeWasTaken@users.noreply.github.com>
+  * Peter Brunner <peter@lugoues.net>
+  * Ninh Pham <dongian.rapclubkhtn@gmail.com>
+  * Ryan Caezar Itang <sitiom@proton.me>
+  * Peter Brunner <peter@psykhe.com>
+  * Leandro Sacchet <leandro.sacchet@animati.com.br>
+  * dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
+  * cycneuramus <56681631+cycneuramus@users.noreply.github.com>
+  * Arnavion <me@arnavion.dev>
+  * Christopher Merry <christopher.merry@mlb.com>
+  * Thibault Coupin <thibault.coupin@gmail.com>
+  * Richard Tweed <RichardoC@users.noreply.github.com>
+  * Zach Kipp <Zacho2@users.noreply.github.com>
+  * yuudi <26199752+yuudi@users.noreply.github.com>
+  * NickIAm <NickIAm@users.noreply.github.com>
+  * Juang, Yi-Lin <frankyjuang@gmail.com>
+  * jumbi77 <jumbi77@users.noreply.github.com>
+  * Aditya Basu <ab.aditya.basu@gmail.com>
+  * ed <s@ocv.me>
+  * Drew Parsons <dparsons@emerall.com>
+  * Joel <joelnb@users.noreply.github.com>
+  * wiserain <mail275@gmail.com>
+  * Roel Arents <roel.arents@kadaster.nl>
+  * Shyim <github@shyim.de>

docs/content/bisync.md

@@ -596,7 +596,7 @@ quashed by adding `--quiet` to the bisync command line.
 # NOTICE: If you make changes to this file you MUST do a --resync run.
 #         Run with --dry-run to see what changes will be made.
 
-# Dropbox wont sync some files so filter them away here.
+# Dropbox won't sync some files so filter them away here.
 # See https://help.dropbox.com/installs-integrations/sync-uploads/files-not-syncing
 - .dropbox.attr
 - ~*.tmp
@@ -1008,7 +1008,7 @@ Your normal workflow might be as follows:
   Delete a single file.
 - `delete-glob <dir> <pattern>`
   Delete a group of files located one level deep in the given directory
-  with names maching a given glob pattern.
+  with names matching a given glob pattern.
 - `touch-glob YYYY-MM-DD <dir> <pattern>`
   Change modification time on a group of files.
 - `touch-copy YYYY-MM-DD <source-file> <dest-dir>`

docs/content/changelog.md

@@ -5,6 +5,129 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.62.2 - 2023-03-16
+
+[See commits](https://github.com/rclone/rclone/compare/v1.62.1...v1.62.2)
+
+* Bug Fixes
+    * docker volume plugin: Add missing fuse3 dependency (Nick Craig-Wood)
+    * docs: Fix size documentation (asdffdsazqqq)
+* FTP
+    * Fix 426 errors on downloads with vsftpd (Lesmiscore)
+
+## v1.62.1 - 2023-03-15
+
+[See commits](https://github.com/rclone/rclone/compare/v1.62.0...v1.62.1)
+
+* Bug Fixes
+    * docker: Add missing fuse3 dependency (cycneuramus)
+    * build: Update release docs to be more careful with the tag (Nick Craig-Wood)
+    * build: Set Github release to draft while uploading binaries (Nick Craig-Wood)
+
+## v1.62.0 - 2023-03-14
+
+[See commits](https://github.com/rclone/rclone/compare/v1.61.0...v1.62.0)
+
+* New Features
+    * accounting: Make checkers show what they are doing (Nick Craig-Wood)
+    * authorize: Add support for custom templates (Hunter Wittenborn)
+    * build
+        * Update to go1.20 (Nick Craig-Wood, Anagh Kumar Baranwal)
+        * Add winget releaser workflow (Ryan Caezar Itang)
+        * Add dependabot (Ryan Caezar Itang)
+    * doc updates (albertony, Bryan Kaplan, Gerard Bosch, IMTheNachoMan, Justin Winokur, Manoj Ghosh, Nick Craig-Wood, Ole Frost, Peter Brunner, piyushgarg, Ryan Caezar Itang, Simmon Li, ToBeFree)
+    * filter: Emit INFO message when can't work out directory filters (Nick Craig-Wood)
+    * fs
+        * Added multiple ca certificate support. (alankrit)
+        * Add `--max-delete-size` a delete size threshold (Leandro Sacchet)
+    * fspath: Allow the symbols `@` and `+` in remote names (albertony)
+    * lib/terminal: Enable windows console virtual terminal sequences processing (ANSI/VT100 colors) (albertony)
+    * move: If `--check-first` and `--order-by` are set then delete with perfect ordering (Nick Craig-Wood)
+    * serve http: Support `--auth-proxy` (Matthias Baur)
+* Bug Fixes
+    * accounting
+        * Avoid negative ETA values for very slow speeds (albertony)
+        * Limit length of ETA string (albertony)
+        * Show human readable elapsed time when longer than a day (albertony)
+    * all: Apply codeql fixes (Aaron Gokaslan)
+    * build
+        * Fix condition for manual workflow run (albertony)
+        * Fix building for ARMv5 and ARMv6 (albertony)
+            * selfupdate: Consider ARM version
+            * install.sh: fix ARMv6 download
+            * version: Report ARM version
+    * deletefile: Return error code 4 if file does not exist (Nick Craig-Wood)
+    * docker: Fix volume plugin does not remount volume on docker restart (logopk)
+    * fs: Fix race conditions in `--max-delete` and `--max-delete-size` (Nick Craig-Wood)
+    * lib/oauthutil: Handle fatal errors better (Alex Chen)
+    * mount2: Fix `--allow-non-empty` (Nick Craig-Wood)
+    * operations: Fix concurrency: use `--checkers` unless transferring files (Nick Craig-Wood)
+    * serve ftp: Fix timestamps older than 1 year in listings (Nick Craig-Wood)
+    * sync: Fix concurrency: use `--checkers` unless transferring files (Nick Craig-Wood)
+    * tree
+        * Fix nil pointer exception on stat failure (Nick Craig-Wood)
+        * Fix colored output on windows (albertony)
+        * Fix display of files with illegal Windows file system names (Nick Craig-Wood)
+* Mount
+    * Fix creating and renaming files on case insensitive backends (Nick Craig-Wood)
+    * Do not treat `\\?\` prefixed paths as network share paths on windows (albertony)
+    * Fix check for empty mount point on Linux (Nick Craig-Wood)
+    * Fix `--allow-non-empty` (Nick Craig-Wood)
+    * Avoid incorrect or premature overlap check on windows (albertony)
+    * Update to fuse3 after bazil.org/fuse update (Nick Craig-Wood)
+* VFS
+    * Make uploaded files retain modtime with non-modtime backends (Nick Craig-Wood)
+    * Fix incorrect modtime on fs which don't support setting modtime (Nick Craig-Wood)
+    * Fix rename of directory containing files to be uploaded (Nick Craig-Wood)
+* Local
+    * Fix `%!w(<nil>)` in "failed to read directory" error (Marks Polakovs)
+    * Fix exclusion of dangling symlinks with -L/--copy-links (Nick Craig-Wood)
+* Crypt
+    * Obey `--ignore-checksum` (Nick Craig-Wood)
+    * Fix for unencrypted directory names on case insensitive remotes (Ole Frost)
+* Azure Blob
+    * Remove workarounds for SDK bugs after v0.6.1 update (Nick Craig-Wood)
+* B2
+    * Fix uploading files bigger than 1TiB (Nick Craig-Wood)
+* Drive
+    * Note that `--drive-acknowledge-abuse` needs SA Manager permission (Nick Craig-Wood)
+    * Make `--drive-stop-on-upload-limit` to respond to storageQuotaExceeded (Ninh Pham)
+* FTP
+    * Retry 426 errors (Nick Craig-Wood)
+    * Retry errors when initiating downloads (Nick Craig-Wood)
+    * Revert to upstream `github.com/jlaffaye/ftp` now fix is merged (Nick Craig-Wood)
+* Google Cloud Storage
+    * Add `--gcs-env-auth` to pick up IAM credentials from env/instance (Peter Brunner)
+* Mega
+    * Add `--mega-use-https` flag (NodudeWasTaken)
+* Onedrive
+    * Default onedrive personal to QuickXorHash as Microsoft is removing SHA1 (Nick Craig-Wood)
+    * Add `--onedrive-hash-type` to change the hash in use (Nick Craig-Wood)
+    * Improve speed of QuickXorHash (LXY)
+* Oracle Object Storage
+    * Speed up operations by using S3 pacer and setting minsleep to 10ms (Manoj Ghosh)
+    * Expose the `storage_tier` option in config (Manoj Ghosh)
+    * Bring your own encryption keys (Manoj Ghosh)
+* S3
+    * Check multipart upload ETag when `--s3-no-head` is in use (Nick Craig-Wood)
+    * Add `--s3-sts-endpoint` to specify STS endpoint (Nick Craig-Wood)
+    * Fix incorrect tier support for StorJ and IDrive when pointing at a file (Ole Frost)
+    * Fix AWS STS failing if `--s3-endpoint` is set (Nick Craig-Wood)
+    * Make purge remove directory markers too (Nick Craig-Wood)
+* Seafile
+    * Renew library password (Fred)
+* SFTP
+    * Fix uploads being 65% slower than they should be with crypt (Nick Craig-Wood)
+* Smb
+    * Allow SPN (service principal name) to be configured (Nick Craig-Wood)
+    * Check smb connection is closed (happyxhw)
+* Storj
+    * Implement `rclone link` (Kaloyan Raev)
+    * Implement `rclone purge` (Kaloyan Raev)
+    * Update satellite urls and labels (Kaloyan Raev)
+* WebDAV
+    * Fix interop with davrods server (Nick Craig-Wood)
+
 ## v1.61.1 - 2022-12-23
 
 [See commits](https://github.com/rclone/rclone/compare/v1.61.0...v1.61.1)
@@ -1897,8 +2020,8 @@ all the docs and Edward Barker for helping re-write the front page.
     * Use proper import path go.etcd.io/bbolt (Robert-André Mauchin)
 * Crypt
     * Calculate hashes for uploads from local disk (Nick Craig-Wood)
-        * This allows crypted Jottacloud uploads without using local disk
-        * This means crypted s3/b2 uploads will now have hashes
+        * This allows encrypted Jottacloud uploads without using local disk
+        * This means encrypted s3/b2 uploads will now have hashes
     * Added `rclone backend decode`/`encode` commands to replicate functionality of `cryptdecode` (Anagh Kumar Baranwal)
     * Get rid of the unused Cipher interface as it obfuscated the code (Nick Craig-Wood)
 * Azure Blob
@@ -3108,7 +3231,7 @@ Point release to fix hubic and azureblob backends.
     * Fix panic when running without plex configs (Remus Bunduc)
     * Fix root folder caching (Remus Bunduc)
 * Crypt
-    * Check the crypted hash of files when uploading for extra data security
+    * Check the encrypted hash of files when uploading for extra data security
 * Dropbox
     * Make Dropbox for business folders accessible using an initial `/` in the path
 * Google Cloud Storage
@@ -3463,7 +3586,7 @@ Point release to fix hubic and azureblob backends.
 * New commands
     * `rcat` - read from standard input and stream upload
     * `tree` - shows a nicely formatted recursive listing
-    * `cryptdecode` - decode crypted file names (thanks ishuah)
+    * `cryptdecode` - decode encrypted file names (thanks ishuah)
     * `config show` - print the config file
     * `config file` - print the config file location
 * New Features
@@ -3767,7 +3890,7 @@ Point release to fix hubic and azureblob backends.
     * Fix `rclone move` command
         * Delete src files which already existed in dst
         * Fix deletion of src file when dst file older
-    * Fix `rclone check` on crypted file systems
+    * Fix `rclone check` on encrypted file systems
     * Make failed uploads not count as "Transferred"
     * Make sure high level retries show with `-q`
     * Use a vendor directory with godep for repeatable builds

docs/content/commands/rclone.md

@@ -47,7 +47,7 @@ See the [global flags page](/flags/) for global options not listed here.
 * [rclone copy](/commands/rclone_copy/)	 - Copy files from source to dest, skipping identical files.
 * [rclone copyto](/commands/rclone_copyto/)	 - Copy files from source to dest, skipping identical files.
 * [rclone copyurl](/commands/rclone_copyurl/)	 - Copy url content to dest.
-* [rclone cryptcheck](/commands/rclone_cryptcheck/)	 - Cryptcheck checks the integrity of a crypted remote.
+* [rclone cryptcheck](/commands/rclone_cryptcheck/)	 - Cryptcheck checks the integrity of an encrypted remote.
 * [rclone cryptdecode](/commands/rclone_cryptdecode/)	 - Cryptdecode returns unencrypted file names.
 * [rclone dedupe](/commands/rclone_dedupe/)	 - Interactively find duplicate filenames and delete/rename them.
 * [rclone delete](/commands/rclone_delete/)	 - Remove the files in path.

docs/content/commands/rclone_authorize.md

@@ -20,9 +20,7 @@ rclone config.
 Use --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.
 
-Use --template to generate HTML output via a custom Go
-template. If a blank string is provided as an argument to
-this flag, the default template is used.
+Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.
 
 ```
 rclone authorize [flags]
@@ -33,7 +31,7 @@ rclone authorize [flags]
 ```
       --auth-no-open-browser   Do not automatically open auth link in default browser
   -h, --help                   help for authorize
-      --template string        Use a custom Go template for generating HTML responses
+      --template string        The path to a custom Go template for generating HTML responses
 ```
 
 See the [global flags page](/flags/) for global options not listed here.

docs/content/commands/rclone_check.md

@@ -18,7 +18,7 @@ match.  It doesn't alter the source or destination.
 
 For the [crypt](/crypt/) remote there is a dedicated command,
 [cryptcheck](/commands/rclone_cryptcheck/), that are able to check
-the checksums of the crypted files.
+the checksums of the encrypted files.
 
 If you supply the `--size-only` flag, it will only compare the sizes not
 the hashes as well.  Use this for a quick check.
@@ -52,6 +52,8 @@ you what happened to it. These are reminiscent of diff files.
 - `* path` means path was present in source and destination but different.
 - `! path` means there was an error reading or hashing the source or dest.
 
+The default number of parallel checks is N=8. See the [--checkers=N](/docs/#checkers-n) option
+for more information.
 
 ```
 rclone check source:path dest:path [flags]

docs/content/commands/rclone_cryptcheck.md

@@ -1,6 +1,6 @@
 ---
 title: "rclone cryptcheck"
-description: "Cryptcheck checks the integrity of a crypted remote."
+description: "Cryptcheck checks the integrity of an encrypted remote."
 slug: rclone_cryptcheck
 url: /commands/rclone_cryptcheck/
 versionIntroduced: v1.36
@@ -8,21 +8,21 @@ versionIntroduced: v1.36
 ---
 # rclone cryptcheck
 
-Cryptcheck checks the integrity of a crypted remote.
+Cryptcheck checks the integrity of an encrypted remote.
 
 ## Synopsis
 
 
-rclone cryptcheck checks a remote against a [crypted](/crypt/) remote.
+rclone cryptcheck checks a remote against an [encrypted](/crypt/) remote.
 This is the equivalent of running rclone [check](/commands/rclone_check/),
-but able to check the checksums of the crypted remote.
+but able to check the checksums of the encrypted remote.
 
-For it to work the underlying remote of the cryptedremote must support
+For it to work the underlying remote of the encryptedremote must support
 some kind of checksum.
 
-It works by reading the nonce from each file on the cryptedremote: and
+It works by reading the nonce from each file on the encryptedremote: and
 using that to encrypt each file on the remote:.  It then checks the
-checksum of the underlying file on the cryptedremote: against the
+checksum of the underlying file on the ercryptedremote: against the
 checksum of the file it has just encrypted.
 
 Use it like this
@@ -57,9 +57,11 @@ you what happened to it. These are reminiscent of diff files.
 - `* path` means path was present in source and destination but different.
 - `! path` means there was an error reading or hashing the source or dest.
 
+The default number of parallel checks is N=8. See the [--checkers=N](/docs/#checkers-n) option
+for more information.
 
 ```
-rclone cryptcheck remote:path cryptedremote:path [flags]
+rclone cryptcheck remote:path encryptedremote:path [flags]
 ```
 
 ## Options