fshttp: increase HTTP/2 buffer size from 16k to 8M to increase througput #8379

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

.github/workflows/build.yml

@@ -26,12 +26,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.22', 'go1.23']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -42,14 +42,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -58,14 +58,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -75,23 +75,23 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.21
-            os: ubuntu-latest
-            go: '1.21'
-            quicktest: true
-            racequicktest: true
-
           - job_name: go1.22
             os: ubuntu-latest
             go: '1.22'
             quicktest: true
             racequicktest: true
 
+          - job_name: go1.23
+            os: ubuntu-latest
+            go: '1.23'
+            quicktest: true
+            racequicktest: true
+
     name: ${{ matrix.job_name }}
 
     runs-on: ${{ matrix.os }}
@@ -311,7 +311,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '>=1.24.0-rc.1'
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_beta_docker_image.yml

@@ -1,77 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKERHUB_USERNAME }}
-                password: ${{ secrets.DOCKERHUB_TOKEN }}
-            - name: Extract metadata (tags, labels) for Docker
-              id: meta
-              uses: docker/metadata-action@v5
-              with:
-                images: ghcr.io/${{ github.repository }}
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to GitHub Container Registry
-              uses: docker/login-action@v3
-              with:
-                registry: ghcr.io
-                # This is the user that triggered the Workflow. In this case, it will
-                # either be the user whom created the Release or manually triggered
-                # the workflow_dispatch.
-                username: ${{ github.actor }}
-                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
-                # GitHub Actions at the start of a workflow run to identify the job.
-                # This is used to authenticate against GitHub Container Registry.
-                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
-                # for more detailed information.
-                password: ${{ secrets.GITHUB_TOKEN }}
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                push: true # push the image to ghcr
-                tags: |
-                  ghcr.io/rclone/rclone:beta
-                  rclone/rclone:beta
-                labels: ${{ steps.meta.outputs.labels }}
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                cache-from: type=gha, scope=${{ github.workflow }}
-                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
-                provenance: false
-              # Eventually cache will need to be cleared if builds more frequent than once a week
-              # https://github.com/docker/build-push-action/issues/252
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,294 @@
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*-
+
+name: Build & Push Docker Images
+
+# Trigger the workflow on push or pull request
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
+
+jobs:
+  build-image:
+    if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runs-on: ubuntu-24.04
+          - platform: linux/386
+            runs-on: ubuntu-24.04
+          - platform: linux/arm64
+            runs-on: ubuntu-24.04-arm
+          - platform: linux/arm/v7
+            runs-on: ubuntu-24.04-arm
+          - platform: linux/arm/v6
+            runs-on: ubuntu-24.04-arm
+
+    name: Build Docker Image for ${{ matrix.platform }}
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - name: Free Space
+        shell: bash
+        run: |
+          df -h .
+          # Remove android SDK
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove .net runtime
+          sudo rm -rf /usr/share/dotnet || true
+          df -h .
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set REPO_NAME Variable
+        run: |
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
+
+      - name: Set PLATFORM Variable
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Set CACHE_NAME Variable
+        shell: python
+        run: |
+          import os, re
+
+          def slugify(input_string, max_length=63):
+            slug = input_string.lower()
+            slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
+            slug = slug.strip()
+            slug = re.sub(r'\s+', '-', slug)
+            slug = re.sub(r'-+', '-', slug)
+            slug = slug[:max_length]
+            slug = re.sub(r'[-]+$', '', slug)
+            return slug
+
+          ref_name_slug = "cache"
+
+          if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
+            ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"CACHE_NAME={ref_name_slug}\n")
+
+      - name: Get ImageOS
+        # There's no way around this, because "ImageOS" is only available to
+        # processes, but the setup-go action uses it in its key.
+        id: imageos
+        uses: actions/github-script@v7
+        with:
+          result-encoding: string
+          script: |
+            return process.env.ImageOS
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
+        with:
+          images: |
+            ghcr.io/${{ env.REPO_NAME }}
+          labels: |
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=beta,enable={{is_default_branch}}
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load Go Build Cache for Docker
+        id: go-cache
+        uses: actions/cache@v4
+        with:
+          key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
+          # Cache only the go builds, the module download is cached via the docker layer caching
+          path: |
+            go-build-cache
+
+      - name: Inject Go Build Cache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "go-build-cache": "/root/.cache/go-build"
+            }
+          skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Publish Image Digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          file: Dockerfile
+          context: .
+          provenance: false
+          # don't specify 'tags' here (error "get can't push tagged ref by digest")
+          # tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
+          platforms: ${{ matrix.platform }}
+          outputs: |
+            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
+          cache-to: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
+
+      - name: Export Image Digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload Image Digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM }}
+          path: /tmp/digests/*
+          retention-days: 1
+          if-no-files-found: error
+
+  merge-image:
+    name: Merge & Push Final Docker Image
+    runs-on: ubuntu-24.04
+    needs:
+      - build-image
+
+    steps:
+      - name: Download Image Digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set REPO_NAME Variable
+        run: |
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
+        with:
+          images: |
+            ${{ env.REPO_NAME }}
+            ghcr.io/${{ env.REPO_NAME }}
+          labels: |
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=beta,enable={{is_default_branch}}
+
+      - name: Extract Tags
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
+          tags_string = " ".join(tags)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"TAGS={tags_string}\n")
+
+      - name: Extract Annotations
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
+          annotations_string = " ".join(annotations)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"ANNOTATIONS={annotations_string}\n")
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create & Push Manifest List
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create \
+          ${{ env.TAGS }} \
+          ${{ env.ANNOTATIONS }} \
+          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
+
+      - name: Inspect and Run Multi-Platform Image
+        run: |
+          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_docker_plugin.yml

@@ -0,0 +1,45 @@
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*-
+
+name: Release Build for Docker Plugin
+
+on:
+  release:
+    types: [published]
+
+jobs:
+
+    build_docker_volume_plugin:
+        if: github.repository == 'rclone/rclone'
+        needs: build
+        runs-on: ubuntu-latest
+        name: Build docker plugin job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Build and publish docker plugin
+              shell: bash
+              run: |
+                VER=${GITHUB_REF#refs/tags/}
+                PLUGIN_USER=rclone
+                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+                    export PLUGIN_USER PLUGIN_ARCH
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+                done
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/build_publish_release_docker_image.yml

@@ -1,89 +0,0 @@
-name: Docker release build
-
-on:
-  release:
-    types: [published]
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Get actual patch version
-              id: actual_patch_version
-              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
-            - name: Get actual minor version
-              id: actual_minor_version
-              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
-            - name: Get actual major version
-              id: actual_major_version
-              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKER_HUB_USER }}
-                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                push: true
-                tags: |
-                  rclone/rclone:latest
-                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
-                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
-                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

Dockerfile

@@ -1,19 +1,47 @@
 FROM golang:alpine AS builder
 
-COPY . /go/src/github.com/rclone/rclone/
+ARG CGO_ENABLED=0
+
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN apk add --no-cache make bash gawk git
-RUN \
-  CGO_ENABLED=0 \
-  make
-RUN ./rclone version
+RUN echo "**** Set Go Environment Variables ****" && \
+    go env -w GOCACHE=/root/.cache/go-build
+
+RUN echo "**** Install Dependencies ****" && \
+    apk add --no-cache \
+        make \
+        bash \
+        gawk \
+        git
+
+COPY go.mod .
+COPY go.sum .
+
+RUN echo "**** Download Go Dependencies ****" && \
+    go mod download -x
+
+RUN echo "**** Verify Go Dependencies ****" && \
+    go mod verify
+
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
+    echo "**** Build Binary ****" && \
+    make
+
+RUN echo "**** Print Version Binary ****" && \
+    ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse3 tzdata && \
-  echo "user_allow_other" >> /etc/fuse.conf
+RUN echo "**** Install Dependencies ****" && \
+    apk add --no-cache \
+        ca-certificates \
+        fuse3 \
+        tzdata && \
+    echo "Enable user_allow_other in fuse" && \
+    echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

RELEASE.md

@@ -47,13 +47,20 @@ Early in the next release cycle update the dependencies.
   * `git commit -a -v -m "build: update all dependencies"`
 
 If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
+
+    go 1.22.0
+    
+then go to manual mode. `go1.22` here is the lowest supported version
 in the `go.mod`.
 
+If `make updatedirect` added a `toolchain` directive then remove it.
+We don't want to force a toolchain on our users. Linux packagers are
+often using a version of Go that is a few versions out of date.
+
 ```
 go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
 go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
+go mod tidy -go=1.22 -compat=1.22
 ```
 
 If the `go mod tidy` fails use the output from it to remove the
@@ -86,6 +93,16 @@ build.
 Once it compiles locally, push it on a test branch and commit fixes
 until the tests pass.
 
+### Major versions
+
+The above procedure will not upgrade major versions, so v2 to v3.
+However this tool can show which major versions might need to be
+upgraded:
+
+    go run github.com/icholy/gomajor@latest list -major
+
+Expect API breakage when updating major versions.
+
 ## Tidy beta
 
 At some point after the release run

VERSION

@@ -1 +1 @@
-v1.69.0
+v1.70.0

backend/all/all.go

@@ -10,6 +10,7 @@ import (
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
+	_ "github.com/rclone/rclone/backend/cloudinary"
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"

backend/azureblob/azureblob_internal_test.go

@@ -3,16 +3,149 @@
 package azureblob
 
 import (
+	"context"
+	"encoding/base64"
+	"strings"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func (f *Fs) InternalTest(t *testing.T) {
-	// Check first feature flags are set on this
-	// remote
+func TestBlockIDCreator(t *testing.T) {
+	// Check creation and random number
+	bic, err := newBlockIDCreator()
+	require.NoError(t, err)
+	bic2, err := newBlockIDCreator()
+	require.NoError(t, err)
+	assert.NotEqual(t, bic.random, bic2.random)
+	assert.NotEqual(t, bic.random, [8]byte{})
+
+	// Set random to known value for tests
+	bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
+	chunkNumber := uint64(0xFEDCBA9876543210)
+
+	// Check creation of ID
+	want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
+	got := bic.newBlockID(chunkNumber)
+	assert.Equal(t, want, got)
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)
+
+	// Test checkID is working
+	assert.NoError(t, bic.checkID(chunkNumber, got))
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
+	assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
+	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
+}
+
+func (f *Fs) testFeatures(t *testing.T) {
+	// Check first feature flags are set on this remote
 	enabled := f.Features().SetTier
 	assert.True(t, enabled)
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+type ReadSeekCloser struct {
+	*strings.Reader
+}
+
+func (r *ReadSeekCloser) Close() error {
+	return nil
+}
+
+// Stage a block at remote but don't commit it
+func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
+	var (
+		containerName, blobPath = f.split(remote)
+		containerClient         = f.cntSVC(containerName)
+		blobClient              = containerClient.NewBlockBlobClient(blobPath)
+		data                    = "uncommitted data"
+		blockID                 = "1"
+		blockIDBase64           = base64.StdEncoding.EncodeToString([]byte(blockID))
+	)
+	r := &ReadSeekCloser{strings.NewReader(data)}
+	_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
+	require.NoError(t, err)
+
+	// Verify the block is staged but not committed
+	blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
+	require.NoError(t, err)
+	found := false
+	for _, block := range blockList.UncommittedBlocks {
+		if *block.Name == blockIDBase64 {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "Block ID not found in uncommitted blocks")
+}
+
+// This tests uploading a blob where it has uncommitted blocks with a different ID size.
+//
+// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
+//
+// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
+func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
+	var (
+		ctx    = context.Background()
+		remote = "testBlob"
+	)
+
+	// Multipart copy the blob please
+	oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
+	f.opt.UseCopyBlob = false
+	f.opt.CopyCutoff = f.opt.ChunkSize
+	defer func() {
+		f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
+	}()
+
+	// Create a blob with uncommitted blocks
+	f.stageBlockWithoutCommit(ctx, t, remote)
+
+	// Now attempt to overwrite the block with a different sized block ID to provoke this error
+
+	// Check the object does not exist
+	_, err := f.NewObject(ctx, remote)
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// Upload a multipart file over the block with uncommitted chunks of a different ID size
+	size := 4*int(f.opt.ChunkSize) - 1
+	contents := random.String(size)
+	item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
+	o := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+
+	// Check size
+	assert.Equal(t, int64(size), o.Size())
+
+	// Create a new blob with uncommitted blocks
+	newRemote := "testBlob2"
+	f.stageBlockWithoutCommit(ctx, t, newRemote)
+
+	// Copy over that block
+	dst, err := f.Copy(ctx, o, newRemote)
+	require.NoError(t, err)
+
+	// Check basics
+	assert.Equal(t, int64(size), dst.Size())
+	assert.Equal(t, newRemote, dst.Remote())
+
+	// Check contents
+	gotContents := fstests.ReadObject(ctx, t, dst, -1)
+	assert.Equal(t, contents, gotContents)
+
+	// Remove the object
+	require.NoError(t, dst.Remove(ctx))
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Features", f.testFeatures)
+	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
+}

backend/azureblob/azureblob_test.go

@@ -15,13 +15,17 @@ import (
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
+	name := "TestAzureBlob"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
+		RemoteName:  name + ":",
 		NilObject:   (*Object)(nil),
 		TiersToTest: []string{"Hot", "Cool", "Cold"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: defaultChunkSize,
 		},
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "use_copy_blob", Value: "false"},
+		},
 	})
 }
 
@@ -40,6 +44,7 @@ func TestIntegration2(t *testing.T) {
 		},
 		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "directory_markers", Value: "true"},
+			{Name: name, Key: "use_copy_blob", Value: "false"},
 		},
 	})
 }
@@ -48,8 +53,13 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }
 
+func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setCopyCutoff(cs)
+}
+
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )
 
 func TestValidateAccessTier(t *testing.T) {

backend/azurefiles/azurefiles.go

@@ -237,6 +237,30 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Help:      "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
 			Advanced:  true,
 			Sensitive: true,
+		}, {
+			Name: "disable_instance_discovery",
+			Help: `Skip requesting Microsoft Entra instance metadata
+This should be set true only by applications authenticating in
+disconnected clouds, or private clouds such as Azure Stack.
+It determines whether rclone requests Microsoft Entra instance
+metadata from ` + "`https://login.microsoft.com/`" + ` before
+authenticating.
+Setting this to true will skip this request, making you responsible
+for ensuring the configured authority is valid and trustworthy.
+`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "use_az",
+			Help: `Use Azure CLI tool az for authentication
+Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
+as the sole means of authentication.
+Setting this can be useful if you wish to use the az CLI on a host with
+a System Managed Identity that you do not want to use.
+Don't set env_auth at the same time.
+`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -319,10 +343,12 @@ type Options struct {
 	Username                   string               `config:"username"`
 	Password                   string               `config:"password"`
 	ServicePrincipalFile       string               `config:"service_principal_file"`
+	DisableInstanceDiscovery   bool                 `config:"disable_instance_discovery"`
 	UseMSI                     bool                 `config:"use_msi"`
 	MSIObjectID                string               `config:"msi_object_id"`
 	MSIClientID                string               `config:"msi_client_id"`
 	MSIResourceID              string               `config:"msi_mi_res_id"`
+	UseAZ                      bool                 `config:"use_az"`
 	Endpoint                   string               `config:"endpoint"`
 	ChunkSize                  fs.SizeSuffix        `config:"chunk_size"`
 	MaxStreamSize              fs.SizeSuffix        `config:"max_stream_size"`
@@ -393,8 +419,10 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 	policyClientOptions := policy.ClientOptions{
 		Transport: newTransporter(ctx),
 	}
+	backup := service.ShareTokenIntentBackup
 	clientOpt := service.ClientOptions{
-		ClientOptions: policyClientOptions,
+		ClientOptions:     policyClientOptions,
+		FileRequestIntent: &backup,
 	}
 
 	// Here we auth by setting one of cred, sharedKeyCred or f.client
@@ -412,7 +440,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		}
 		// Read credentials from the environment
 		options := azidentity.DefaultAzureCredentialOptions{
-			ClientOptions: policyClientOptions,
+			ClientOptions:            policyClientOptions,
+			DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
 		}
 		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
@@ -423,6 +452,13 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		if err != nil {
 			return nil, fmt.Errorf("create new shared key credential failed: %w", err)
 		}
+	case opt.UseAZ:
+		var options = azidentity.AzureCLICredentialOptions{}
+		cred, err = azidentity.NewAzureCLICredential(&options)
+		fmt.Println(cred)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
+		}
 	case opt.SASURL != "":
 		client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt)
 		if err != nil {
@@ -897,7 +933,7 @@ func (o *Object) getMetadata(ctx context.Context) error {
 
 // Hash returns the MD5 of an object returning a lowercase hex string
 //
-// May make a network request becaue the [fs.List] method does not
+// May make a network request because the [fs.List] method does not
 // return MD5 hashes for DirEntry
 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
 	if ty != hash.MD5 {

backend/b2/api/types.go

@@ -42,9 +42,10 @@ type Bucket struct {
 
 // LifecycleRule is a single lifecycle rule
 type LifecycleRule struct {
-	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
-	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
-	FileNamePrefix            string `json:"fileNamePrefix"`
+	DaysFromHidingToDeleting                        *int   `json:"daysFromHidingToDeleting"`
+	DaysFromUploadingToHiding                       *int   `json:"daysFromUploadingToHiding"`
+	DaysFromStartingToCancelingUnfinishedLargeFiles *int   `json:"daysFromStartingToCancelingUnfinishedLargeFiles"`
+	FileNamePrefix                                  string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base

backend/b2/b2.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
@@ -299,14 +300,13 @@ type Fs struct {
 
 // Object describes a b2 object
 type Object struct {
-	fs       *Fs               // what this object is part of
-	remote   string            // The remote path
-	id       string            // b2 id of the file
-	modTime  time.Time         // The modified time of the object if known
-	sha1     string            // SHA-1 hash if known
-	size     int64             // Size of the object
-	mimeType string            // Content-Type of the object
-	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
+	fs       *Fs       // what this object is part of
+	remote   string    // The remote path
+	id       string    // b2 id of the file
+	modTime  time.Time // The modified time of the object if known
+	sha1     string    // SHA-1 hash if known
+	size     int64     // Size of the object
+	mimeType string    // Content-Type of the object
 }
 
 // ------------------------------------------------------------
@@ -1318,16 +1318,22 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 				// Check current version of the file
 				if deleteHidden && object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove hide marker") {
+						toBeDeleted <- object
+					}
 				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove pending upload") {
+						toBeDeleted <- object
+					}
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
-				toBeDeleted <- object
+				if !operations.SkipDestructive(ctx, object.Name, "delete") {
+					toBeDeleted <- object
+				}
 			}
 			last = remote
 			tr.Done(ctx, nil)
@@ -1598,9 +1604,6 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if err != nil {
 		return err
 	}
-	// For now, just set "mtime" in metadata
-	o.meta = make(map[string]string, 1)
-	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
 	return nil
 }
 
@@ -1880,13 +1883,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		Info:            Info,
 	}
 
-	// Embryonic metadata support - just mtime
-	o.meta = make(map[string]string, 1)
-	modTime, err := parseTimeStringHelper(info.Info[timeKey])
-	if err == nil {
-		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
-	}
-
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -2231,6 +2227,7 @@ This will dump something like this showing the lifecycle rules.
         {
             "daysFromHidingToDeleting": 1,
             "daysFromUploadingToHiding": null,
+            "daysFromStartingToCancelingUnfinishedLargeFiles": null,
             "fileNamePrefix": ""
         }
     ]
@@ -2257,8 +2254,9 @@ overwrites will still cause versions to be made.
 See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 `,
 	Opts: map[string]string{
-		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
-		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
+		"daysFromHidingToDeleting":                        "After a file has been hidden for this many days it is deleted. 0 is off.",
+		"daysFromUploadingToHiding":                       "This many days after uploading a file is hidden",
+		"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
 	},
 }
 
@@ -2278,14 +2276,23 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 		}
 		newRule.DaysFromUploadingToHiding = &days
 	}
+	if daysStr := opt["daysFromStartingToCancelingUnfinishedLargeFiles"]; daysStr != "" {
+		days, err := strconv.Atoi(daysStr)
+		if err != nil {
+			return nil, fmt.Errorf("bad daysFromStartingToCancelingUnfinishedLargeFiles: %w", err)
+		}
+		newRule.DaysFromStartingToCancelingUnfinishedLargeFiles = &days
+	}
 	bucketName, _ := f.split("")
 	if bucketName == "" {
 		return nil, errors.New("bucket required")
 
 	}
 
+	skip := operations.SkipDestructive(ctx, name, "update lifecycle rules")
+
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
+	if !skip && (newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil) {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err

backend/b2/b2_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -13,6 +14,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/bucket"
@@ -256,12 +258,6 @@ func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string
 			assert.Equal(t, v, got, k)
 		}
 
-		// mtime
-		for k, v := range metadata {
-			got := o.meta[k]
-			assert.Equal(t, v, got, k)
-		}
-
 		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
 
 		// Modification time from the x-bz-info-src_last_modified_millis header
@@ -463,24 +459,161 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	})
 
 	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
+		t.Run("DryRun", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should be unchanged after dry run
+			before := listAllFiles(ctx, t, f, dirName)
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			after := listAllFiles(ctx, t, f, dirName)
+			assert.Equal(t, before, after)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should reflect current state after cleanup
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+			fstest.CheckListing(t, f, items)
+		})
 	})
 
 	// Purge gets tested later
 }
 
+func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
+	ctx := context.Background()
+
+	// B2CleanupHidden tests cleaning up hidden files
+	t.Run("CleanupUnfinished", func(t *testing.T) {
+		dirName := "unfinished"
+		fileCount := 5
+		expectedFiles := []string{}
+		for i := 1; i < fileCount; i++ {
+			fileName := fmt.Sprintf("%s/unfinished-%d", dirName, i)
+			expectedFiles = append(expectedFiles, fileName)
+			obj := &Object{
+				fs:     f,
+				remote: fileName,
+			}
+			objInfo := object.NewStaticObjectInfo(fileName, fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
+			_, err := f.newLargeUpload(ctx, obj, nil, objInfo, f.opt.ChunkSize, false, nil)
+			require.NoError(t, err)
+		}
+		checkListing(ctx, t, f, dirName, expectedFiles)
+
+		t.Run("DryRun", func(t *testing.T) {
+			// Listing should not change after dry run
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, expectedFiles)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			// Listing should be empty after real cleanup
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, []string{})
+		})
+	})
+}
+
+func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
+	bucket, directory := f.split(dirName)
+	foundFiles := []string{}
+	require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
+		if !isDirectory {
+			foundFiles = append(foundFiles, object.Name)
+		}
+		return nil
+	}))
+	sort.Strings(foundFiles)
+	return foundFiles
+}
+
+func checkListing(ctx context.Context, t *testing.T, f *Fs, dirName string, expectedFiles []string) {
+	foundFiles := listAllFiles(ctx, t, f, dirName)
+	sort.Strings(expectedFiles)
+	assert.Equal(t, expectedFiles, foundFiles)
+}
+
+func (f *Fs) InternalTestLifecycleRules(t *testing.T) {
+	ctx := context.Background()
+
+	opt := map[string]string{}
+
+	t.Run("InitState", func(t *testing.T) {
+		// There should be no lifecycle rules at the outset
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("DryRun", func(t *testing.T) {
+		// There should still be no lifecycle rules after each dry run operation
+		ctx, ci := fs.AddConfig(ctx)
+		ci.DryRun = true
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("RealThing", func(t *testing.T) {
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+	})
+}
+
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("Versions", f.InternalTestVersions)
+	t.Run("CleanupUnfinished", f.InternalTestCleanupUnfinished)
+	t.Run("LifecycleRules", f.InternalTestLifecycleRules)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/box/box.go

@@ -255,6 +255,9 @@ func getQueryParams(boxConfig *api.ConfigJSON) map[string]string {
 
 func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) {
 	block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey))
+	if block == nil {
+		return nil, errors.New("box: failed to PEM decode private key")
+	}
 	if len(rest) > 0 {
 		return nil, fmt.Errorf("box: extra data included in private key: %w", err)
 	}

backend/chunker/chunker.go

@@ -2480,7 +2480,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	if len(data) > maxMetadataSizeWritten {
 		return nil, false, ErrMetaTooBig
 	}
-	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
+	if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
 		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON

backend/cloudinary/api/types.go

@@ -0,0 +1,48 @@
+// Package api has type definitions for cloudinary
+package api
+
+import (
+	"fmt"
+)
+
+// CloudinaryEncoder extends the built-in encoder
+type CloudinaryEncoder interface {
+	// FromStandardPath takes a / separated path in Standard encoding
+	// and converts it to a / separated path in this encoding.
+	FromStandardPath(string) string
+	// FromStandardName takes name in Standard encoding and converts
+	// it in this encoding.
+	FromStandardName(string) string
+	// ToStandardPath takes a / separated path in this encoding
+	// and converts it to a / separated path in Standard encoding.
+	ToStandardPath(string) string
+	// ToStandardName takes name in this encoding and converts
+	// it in Standard encoding.
+	ToStandardName(string) string
+	// Encoded root of the remote (as passed into NewFs)
+	FromStandardFullPath(string) string
+}
+
+// UpdateOptions was created to pass options from Update to Put
+type UpdateOptions struct {
+	PublicID     string
+	ResourceType string
+	DeliveryType string
+	AssetFolder  string
+	DisplayName  string
+}
+
+// Header formats the option as a string
+func (o *UpdateOptions) Header() (string, string) {
+	return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *UpdateOptions) Mandatory() bool {
+	return false
+}
+
+// String formats the option into human-readable form
+func (o *UpdateOptions) String() string {
+	return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
+}

backend/cloudinary/cloudinary.go

@@ -0,0 +1,711 @@
+// Package cloudinary provides an interface to the Cloudinary DAM
+package cloudinary
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudinary/cloudinary-go/v2"
+	SDKApi "github.com/cloudinary/cloudinary-go/v2/api"
+	"github.com/cloudinary/cloudinary-go/v2/api/admin"
+	"github.com/cloudinary/cloudinary-go/v2/api/admin/search"
+	"github.com/cloudinary/cloudinary-go/v2/api/uploader"
+	"github.com/rclone/rclone/backend/cloudinary/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/zeebo/blake3"
+)
+
+// Cloudinary shouldn't have a trailing dot if there is no path
+func cldPathDir(somePath string) string {
+	if somePath == "" || somePath == "." {
+		return somePath
+	}
+	dir := path.Dir(somePath)
+	if dir == "." {
+		return ""
+	}
+	return dir
+}
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "cloudinary",
+		Description: "Cloudinary",
+		NewFs:       NewFs,
+		Options: []fs.Option{
+			{
+				Name:      "cloud_name",
+				Help:      "Cloudinary Environment Name",
+				Required:  true,
+				Sensitive: true,
+			},
+			{
+				Name:      "api_key",
+				Help:      "Cloudinary API Key",
+				Required:  true,
+				Sensitive: true,
+			},
+			{
+				Name:      "api_secret",
+				Help:      "Cloudinary API Secret",
+				Required:  true,
+				Sensitive: true,
+			},
+			{
+				Name: "upload_prefix",
+				Help: "Specify the API endpoint for environments out of the US",
+			},
+			{
+				Name: "upload_preset",
+				Help: "Upload Preset to select asset manipulation on upload",
+			},
+			{
+				Name:     config.ConfigEncoding,
+				Help:     config.ConfigEncodingHelp,
+				Advanced: true,
+				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
+					encoder.EncodeSlash |
+					encoder.EncodeLtGt |
+					encoder.EncodeDoubleQuote |
+					encoder.EncodeQuestion |
+					encoder.EncodeAsterisk |
+					encoder.EncodePipe |
+					encoder.EncodeHash |
+					encoder.EncodePercent |
+					encoder.EncodeBackSlash |
+					encoder.EncodeDel |
+					encoder.EncodeCtl |
+					encoder.EncodeRightSpace |
+					encoder.EncodeInvalidUtf8 |
+					encoder.EncodeDot),
+			},
+			{
+				Name:     "eventually_consistent_delay",
+				Default:  fs.Duration(0),
+				Advanced: true,
+				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",
+			},
+		},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	CloudName                 string               `config:"cloud_name"`
+	APIKey                    string               `config:"api_key"`
+	APISecret                 string               `config:"api_secret"`
+	UploadPrefix              string               `config:"upload_prefix"`
+	UploadPreset              string               `config:"upload_preset"`
+	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EventuallyConsistentDelay fs.Duration          `config:"eventually_consistent_delay"`
+}
+
+// Fs represents a remote cloudinary server
+type Fs struct {
+	name     string
+	root     string
+	opt      Options
+	features *fs.Features
+	pacer    *fs.Pacer
+	srv      *rest.Client           // For downloading assets via the Cloudinary CDN
+	cld      *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK
+	lastCRUD time.Time
+}
+
+// Object describes a cloudinary object
+type Object struct {
+	fs           *Fs
+	remote       string
+	size         int64
+	modTime      time.Time
+	url          string
+	md5sum       string
+	publicID     string
+	resourceType string
+	deliveryType string
+}
+
+// NewFs constructs an Fs from the path, bucket:path
+func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize the Cloudinary client
+	cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)
+	}
+	cld.Admin.Client = *fshttp.NewClient(ctx)
+	cld.Upload.Client = *fshttp.NewClient(ctx)
+	if opt.UploadPrefix != "" {
+		cld.Config.API.UploadPrefix = opt.UploadPrefix
+	}
+	client := fshttp.NewClient(ctx)
+	f := &Fs{
+		name:  name,
+		root:  root,
+		opt:   *opt,
+		cld:   cld,
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
+		srv:   rest.NewClient(client),
+	}
+
+	f.features = (&fs.Features{
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f)
+
+	if root != "" {
+		// Check to see if the root actually an existing file
+		remote := path.Base(root)
+		f.root = cldPathDir(root)
+		_, err := f.NewObject(ctx, remote)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
+				// File doesn't exist so return the previous root
+				f.root = root
+				return f, nil
+			}
+			return nil, err
+		}
+		// return an error with an fs which points to the parent
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// ------------------------------------------------------------
+
+// FromStandardPath implementation of the api.CloudinaryEncoder
+func (f *Fs) FromStandardPath(s string) string {
+	return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")
+}
+
+// FromStandardName implementation of the api.CloudinaryEncoder
+func (f *Fs) FromStandardName(s string) string {
+	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
+}
+
+// ToStandardPath implementation of the api.CloudinaryEncoder
+func (f *Fs) ToStandardPath(s string) string {
+	return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")
+}
+
+// ToStandardName implementation of the api.CloudinaryEncoder
+func (f *Fs) ToStandardName(s string) string {
+	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
+}
+
+// FromStandardFullPath encodes a full path to Cloudinary standard
+func (f *Fs) FromStandardFullPath(dir string) string {
+	return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))
+}
+
+// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK
+func (f *Fs) ToAssetFolderAPI(dir string) string {
+	return strings.ReplaceAll(dir, "%", "%25")
+}
+
+// ToDisplayNameElastic encodes a special case of elasticsearch
+func (f *Fs) ToDisplayNameElastic(dir string) string {
+	return strings.ReplaceAll(dir, "!", "\\!")
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// WaitEventuallyConsistent waits till the FS is eventually consistent
+func (f *Fs) WaitEventuallyConsistent() {
+	if f.opt.EventuallyConsistentDelay == fs.Duration(0) {
+		return
+	}
+	delay := time.Duration(f.opt.EventuallyConsistentDelay)
+	timeSinceLastCRUD := time.Since(f.lastCRUD)
+	if timeSinceLastCRUD < delay {
+		time.Sleep(delay - timeSinceLastCRUD)
+	}
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Cloudinary root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// List the objects and directories in dir into entries
+func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
+	remotePrefix := f.FromStandardFullPath(dir)
+	if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {
+		remotePrefix += "/"
+	}
+
+	var entries fs.DirEntries
+	dirs := make(map[string]struct{})
+	nextCursor := ""
+	f.WaitEventuallyConsistent()
+	for {
+		// user the folders api to list folders.
+		folderParams := admin.SubFoldersParams{
+			Folder:     f.ToAssetFolderAPI(remotePrefix),
+			MaxResults: 500,
+		}
+		if nextCursor != "" {
+			folderParams.NextCursor = nextCursor
+		}
+
+		results, err := f.cld.Admin.SubFolders(ctx, folderParams)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list sub-folders: %w", err)
+		}
+		if results.Error.Message != "" {
+			if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {
+				return nil, fs.ErrorDirNotFound
+			}
+
+			return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)
+		}
+
+		for _, folder := range results.Folders {
+			relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))
+			parts := strings.Split(relativePath, "/")
+
+			// It's a directory
+			dirName := parts[len(parts)-1]
+			if _, found := dirs[dirName]; !found {
+				d := fs.NewDir(path.Join(dir, dirName), time.Time{})
+				entries = append(entries, d)
+				dirs[dirName] = struct{}{}
+			}
+		}
+		// Break if there are no more results
+		if results.NextCursor == "" {
+			break
+		}
+		nextCursor = results.NextCursor
+	}
+
+	for {
+		// Use the assets.AssetsByAssetFolder API to list assets
+		assetsParams := admin.AssetsByAssetFolderParams{
+			AssetFolder: remotePrefix,
+			MaxResults:  500,
+		}
+		if nextCursor != "" {
+			assetsParams.NextCursor = nextCursor
+		}
+
+		results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list assets: %w", err)
+		}
+
+		for _, asset := range results.Assets {
+			remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
+			if dir != "" {
+				remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
+			}
+			o := &Object{
+				fs:           f,
+				remote:       remote,
+				size:         int64(asset.Bytes),
+				modTime:      asset.CreatedAt,
+				url:          asset.SecureURL,
+				publicID:     asset.PublicID,
+				resourceType: asset.AssetType,
+				deliveryType: asset.Type,
+			}
+			entries = append(entries, o)
+		}
+
+		// Break if there are no more results
+		if results.NextCursor == "" {
+			break
+		}
+		nextCursor = results.NextCursor
+	}
+
+	return entries, nil
+}
+
+// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	searchParams := search.Query{
+		Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",
+			f.FromStandardFullPath(cldPathDir(remote)),
+			f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),
+		SortBy:     []search.SortByField{{"uploaded_at": "desc"}},
+		MaxResults: 2,
+	}
+	var results *admin.SearchResult
+	f.WaitEventuallyConsistent()
+	err := f.pacer.Call(func() (bool, error) {
+		var err1 error
+		results, err1 = f.cld.Admin.Search(ctx, searchParams)
+		if err1 == nil && results.TotalCount != len(results.Assets) {
+			err1 = errors.New("partial response so waiting for eventual consistency")
+		}
+		return shouldRetry(ctx, nil, err1)
+	})
+	if err != nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	if results.TotalCount == 0 || len(results.Assets) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	asset := results.Assets[0]
+
+	o := &Object{
+		fs:           f,
+		remote:       remote,
+		size:         int64(asset.Bytes),
+		modTime:      asset.UploadedAt,
+		url:          asset.SecureURL,
+		md5sum:       asset.Etag,
+		publicID:     asset.PublicID,
+		resourceType: asset.ResourceType,
+		deliveryType: asset.Type,
+	}
+
+	return o, nil
+}
+
+func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {
+	payload := []byte(path.Join(assetFolder, displayName))
+	hash := blake3.Sum256(payload)
+	return hex.EncodeToString(hash[:])
+}
+
+// Put uploads content to Cloudinary
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	if src.Size() == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
+
+	params := uploader.UploadParams{
+		UploadPreset: f.opt.UploadPreset,
+	}
+
+	updateObject := false
+	var modTime time.Time
+	for _, option := range options {
+		if updateOptions, ok := option.(*api.UpdateOptions); ok {
+			if updateOptions.PublicID != "" {
+				updateObject = true
+				params.Overwrite = SDKApi.Bool(true)
+				params.Invalidate = SDKApi.Bool(true)
+				params.PublicID = updateOptions.PublicID
+				params.ResourceType = updateOptions.ResourceType
+				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
+				params.AssetFolder = updateOptions.AssetFolder
+				params.DisplayName = updateOptions.DisplayName
+				modTime = src.ModTime(ctx)
+			}
+		}
+	}
+	if !updateObject {
+		params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))
+		params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))
+		// We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).
+		// We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.
+		// Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.
+		params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))
+	}
+	uploadResult, err := f.cld.Upload.Upload(ctx, in, params)
+	f.lastCRUD = time.Now()
+	if err != nil {
+		return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)
+	}
+	if !updateObject {
+		modTime = uploadResult.CreatedAt
+	}
+	if uploadResult.Error.Message != "" {
+		return nil, errors.New(uploadResult.Error.Message)
+	}
+
+	o := &Object{
+		fs:           f,
+		remote:       src.Remote(),
+		size:         int64(uploadResult.Bytes),
+		modTime:      modTime,
+		url:          uploadResult.SecureURL,
+		md5sum:       uploadResult.Etag,
+		publicID:     uploadResult.PublicID,
+		resourceType: uploadResult.ResourceType,
+		deliveryType: uploadResult.Type,
+	}
+	return o, nil
+}
+
+// Precision of the remote
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns the supported hash sets
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.MD5)
+}
+
+// Mkdir creates empty folders
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}
+	res, err := f.cld.Admin.CreateFolder(ctx, params)
+	f.lastCRUD = time.Now()
+	if err != nil {
+		return err
+	}
+	if res.Error.Message != "" {
+		return errors.New(res.Error.Message)
+	}
+
+	return nil
+}
+
+// Rmdir deletes empty folders
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	// Additional test because Cloudinary will delete folders without
+	// assets, regardless of empty sub-folders
+	folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))
+	folderParams := admin.SubFoldersParams{
+		Folder:     folder,
+		MaxResults: 1,
+	}
+	results, err := f.cld.Admin.SubFolders(ctx, folderParams)
+	if err != nil {
+		return err
+	}
+	if results.TotalCount > 0 {
+		return fs.ErrorDirectoryNotEmpty
+	}
+
+	params := admin.DeleteFolderParams{Folder: folder}
+	res, err := f.cld.Admin.DeleteFolder(ctx, params)
+	f.lastCRUD = time.Now()
+	if err != nil {
+		return err
+	}
+	if res.Error.Message != "" {
+		if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {
+			return fs.ErrorDirNotFound
+		}
+
+		return errors.New(res.Error.Message)
+	}
+
+	return nil
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	420, // Too Many Requests (legacy)
+	429, // Too Many Requests
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+	509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	if err != nil {
+		tryAgain := "Try again on "
+		if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
+			layout := "2006-01-02 15:04:05 UTC"
+			dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
+			timestamp, err2 := time.Parse(layout, dateStr)
+			if err2 == nil {
+				return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
+			}
+		}
+
+		fs.Debugf(nil, "Retrying API error %v", err)
+		return true, err
+	}
+
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// ------------------------------------------------------------
+
+// Hash returns the MD5 of an object
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
+	if ty != hash.MD5 {
+		return "", hash.ErrUnsupported
+	}
+	return o.md5sum, nil
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// Size of object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// Storable returns if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	var resp *http.Response
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: o.url,
+		Options: options,
+	}
+	var offset int64
+	var count int64
+	var key string
+	var value string
+	fs.FixRangeOption(options, o.size)
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, count = x.Decode(o.size)
+			if count < 0 {
+				count = o.size - offset
+			}
+			key, value = option.Header()
+		case *fs.SeekOption:
+			offset = x.Offset
+			count = o.size - offset
+			key, value = option.Header()
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	if key != "" && value != "" {
+		opts.ExtraHeaders = make(map[string]string)
+		opts.ExtraHeaders[key] = value
+	}
+	// Make sure that the asset is fully available
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		if err == nil {
+			cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))
+			if clErr == nil && count == int64(cl) {
+				return false, nil
+			}
+		}
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)
+	}
+	return resp.Body, err
+}
+
+// Update the object with the contents of the io.Reader
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	options = append(options, &api.UpdateOptions{
+		PublicID:     o.publicID,
+		ResourceType: o.resourceType,
+		DeliveryType: o.deliveryType,
+		DisplayName:  api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),
+		AssetFolder:  o.fs.FromStandardFullPath(cldPathDir(o.Remote())),
+	})
+	updatedObj, err := o.fs.Put(ctx, in, src, options...)
+	if err != nil {
+		return err
+	}
+	if uo, ok := updatedObj.(*Object); ok {
+		o.size = uo.size
+		o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time
+		o.url = uo.url
+		o.md5sum = uo.md5sum
+		o.publicID = uo.publicID
+		o.resourceType = uo.resourceType
+		o.deliveryType = uo.deliveryType
+	}
+	return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	params := uploader.DestroyParams{
+		PublicID:     o.publicID,
+		ResourceType: o.resourceType,
+		Type:         o.deliveryType,
+	}
+	res, dErr := o.fs.cld.Upload.Destroy(ctx, params)
+	o.fs.lastCRUD = time.Now()
+	if dErr != nil {
+		return dErr
+	}
+
+	if res.Error.Message != "" {
+		return errors.New(res.Error.Message)
+	}
+
+	if res.Result != "ok" {
+		return errors.New(res.Result)
+	}
+
+	return nil
+}

backend/cloudinary/cloudinary_test.go

@@ -0,0 +1,23 @@
+// Test Cloudinary filesystem interface
+
+package cloudinary_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/cloudinary"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	name := "TestCloudinary"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:      name + ":",
+		NilObject:       (*cloudinary.Object)(nil),
+		SkipInvalidUTF8: true,
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "eventually_consistent_delay", Value: "7"},
+		},
+	})
+}

backend/drive/drive.go

@@ -203,7 +203,6 @@ func driveScopesContainsAppFolder(scopes []string) bool {
 		if scope == scopePrefix+"drive.appfolder" {
 			return true
 		}
-
 	}
 	return false
 }
@@ -1212,6 +1211,7 @@ func fixMimeType(mimeTypeIn string) string {
 	}
 	return mimeTypeOut
 }
+
 func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	out = make(map[string][]string, len(in))
 	for k, v := range in {
@@ -1222,9 +1222,11 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	}
 	return out
 }
+
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
 }
+
 func isLinkMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/x-link-")
 }
@@ -1657,7 +1659,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	ctx context.Context, remote string, info *drive.File,
-	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
+	extension, exportName, exportMimeType string, isDocument bool,
+) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
@@ -1848,6 +1851,7 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
+
 func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
@@ -1893,7 +1897,8 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
 func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
-	extension, mimeType string, isDocument bool) {
+	extension, mimeType string, isDocument bool,
+) {
 	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
@@ -2689,7 +2694,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if shortcutID != "" {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
-	var trashedFiles = false
+	trashedFiles := false
 	if check {
 		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
 			if !item.Trashed {
@@ -2926,7 +2931,6 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
-
 	if err != nil {
 		return err
 	}
@@ -3187,6 +3191,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
+
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
@@ -3525,14 +3530,14 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 	return f.unTrash(ctx, dir, directoryID, true)
 }
 
-// copy file with id to dest
-func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
+// copy or move file with id to dest
+func (f *Fs) copyOrMoveID(ctx context.Context, operation string, id, dest string) (err error) {
 	info, err := f.getFile(ctx, id, f.getFileFields(ctx))
 	if err != nil {
 		return fmt.Errorf("couldn't find id: %w", err)
 	}
 	if info.MimeType == driveFolderType {
-		return fmt.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
+		return fmt.Errorf("can't %s directory use: rclone %s --drive-root-folder-id %s %s %s", operation, operation, id, fs.ConfigString(f), dest)
 	}
 	info.Name = f.opt.Enc.ToStandardName(info.Name)
 	o, err := f.newObjectWithInfo(ctx, info.Name, info)
@@ -3553,9 +3558,15 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = operations.Copy(ctx, dstFs, nil, destLeaf, o)
-	if err != nil {
-		return fmt.Errorf("copy failed: %w", err)
+
+	var opErr error
+	if operation == "moveid" {
+		_, opErr = operations.Move(ctx, dstFs, nil, destLeaf, o)
+	} else {
+		_, opErr = operations.Copy(ctx, dstFs, nil, destLeaf, o)
+	}
+	if opErr != nil {
+		return fmt.Errorf("%s failed: %w", operation, opErr)
 	}
 	return nil
 }
@@ -3792,6 +3803,28 @@ attempted if possible.
 
 Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
 `,
+}, {
+	Name:  "moveid",
+	Short: "Move files by ID",
+	Long: `This command moves files by ID
+
+Usage:
+
+    rclone backend moveid drive: ID path
+    rclone backend moveid drive: ID1 path1 ID2 path2
+
+It moves the drive file with ID given to the path (an rclone path which
+will be passed internally to rclone moveto).
+
+The path should end with a / to indicate move the file as named to
+this directory. If it doesn't end with a / then the last path
+component will be used as the file name.
+
+If the destination is a drive backend then server-side moving will be
+attempted if possible.
+
+Use the --interactive/-i or --dry-run flag to see what would be moved beforehand.
+`,
 }, {
 	Name:  "exportformats",
 	Short: "Dump the export formats for debug purposes",
@@ -3970,16 +4003,16 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			dir = arg[0]
 		}
 		return f.unTrashDir(ctx, dir, true)
-	case "copyid":
+	case "copyid", "moveid":
 		if len(arg)%2 != 0 {
 			return nil, errors.New("need an even number of arguments")
 		}
 		for len(arg) > 0 {
 			id, dest := arg[0], arg[1]
 			arg = arg[2:]
-			err = f.copyID(ctx, id, dest)
+			err = f.copyOrMoveID(ctx, name, id, dest)
 			if err != nil {
-				return nil, fmt.Errorf("failed copying %q to %q: %w", id, dest, err)
+				return nil, fmt.Errorf("failed %s %q to %q: %w", name, id, dest, err)
 			}
 		}
 		return nil, nil
@@ -3990,14 +4023,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	case "query":
 		if len(arg) == 1 {
 			query := arg[0]
-			var results, err = f.query(ctx, query)
+			results, err := f.query(ctx, query)
 			if err != nil {
 				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
 			}
 			return results, nil
-		} else {
-			return nil, errors.New("need a query argument")
 		}
+		return nil, errors.New("need a query argument")
 	case "rescue":
 		dirID := ""
 		_, delete := opt["delete"]
@@ -4057,6 +4089,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	return "", hash.ErrUnsupported
 }
+
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
 		return "", hash.ErrUnsupported
@@ -4071,7 +4104,8 @@ func (o *baseObject) Size() int64 {
 
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
-	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
+	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error,
+) {
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -4284,12 +4318,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	return o.baseObject.open(ctx, o.url, options...)
 }
+
 func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
 	var offset, end int64 = 0, -1
-	var newOptions = options[:0]
+	newOptions := options[:0]
 	for _, o := range options {
 		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
@@ -4316,9 +4351,10 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
 	}
 	return
 }
+
 func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
-	var data = o.content
+	data := o.content
 	for _, option := range options {
 		switch x := option.(type) {
 		case *fs.SeekOption:
@@ -4343,7 +4379,8 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
-	src fs.ObjectInfo) (info *drive.File, err error) {
+	src fs.ObjectInfo,
+) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
 	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
@@ -4421,6 +4458,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	return nil
 }
+
 func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	srcMimeType := fs.MimeType(ctx, src)
 	importMimeType := ""
@@ -4516,6 +4554,7 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
+
 func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }

backend/drive/drive_internal_test.go

@@ -479,8 +479,8 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) {
 	require.NoError(t, f.Purge(ctx, "trashDir"))
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
+// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyOrMoveID
+func (f *Fs) InternalTestCopyOrMoveID(t *testing.T) {
 	ctx := context.Background()
 	obj, err := f.NewObject(ctx, existingFile)
 	require.NoError(t, err)
@@ -498,7 +498,7 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	}
 
 	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", "ID-NOT-FOUND", dir+"/")
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "couldn't find id")
 	})
@@ -506,19 +506,31 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	t.Run("Directory", func(t *testing.T) {
 		rootID, err := f.dirCache.RootID(ctx, false)
 		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", rootID, dir+"/")
 		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
+		assert.Contains(t, err.Error(), "can't moveid directory")
 	})
 
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
+	t.Run("MoveWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/")
 		require.NoError(t, err)
 		checkFile(path.Base(existingFile))
 	})
 
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
+	t.Run("CopyWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/")
+		require.NoError(t, err)
+		checkFile(path.Base(existingFile))
+	})
+
+	t.Run("MoveWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/potato.txt")
+		require.NoError(t, err)
+		checkFile("potato.txt")
+	})
+
+	t.Run("CopyWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/potato.txt")
 		require.NoError(t, err)
 		checkFile("potato.txt")
 	})
@@ -647,7 +659,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	})
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
+	t.Run("CopyOrMoveID", f.InternalTestCopyOrMoveID)
 	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)

backend/dropbox/batcher.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
-	"github.com/rclone/rclone/fs/fserrors"
 )
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
@@ -21,14 +20,10 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
-		// If error is insufficient space then don't retry
-		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
-			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
-				err = fserrors.NoRetryError(err)
-				return false, err
-			}
+		if retry, err := shouldRetryExclude(ctx, err); !retry {
+			return retry, err
 		}
-		// after the first chunk is uploaded, we retry everything
+		// after the first chunk is uploaded, we retry everything except the excluded errors
 		return err != nil, err
 	})
 	if err != nil {

backend/dropbox/dropbox.go

@@ -318,32 +318,46 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
+// Some specific errors which should be excluded from retries
+func shouldRetryExclude(ctx context.Context, err error) (bool, error) {
 	if err == nil {
 		return false, err
 	}
-	errString := err.Error()
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
 	// First check for specific errors
+	//
+	// These come back from the SDK in a whole host of different
+	// error types, but there doesn't seem to be a consistent way
+	// of reading the error cause, so here we just check using the
+	// error string which isn't perfect but does the job.
+	errString := err.Error()
 	if strings.Contains(errString, "insufficient_space") {
 		return false, fserrors.FatalError(err)
 	} else if strings.Contains(errString, "malformed_path") {
 		return false, fserrors.NoRetryError(err)
 	}
+	return true, err
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if retry, err := shouldRetryExclude(ctx, err); !retry {
+		return retry, err
+	}
 	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
-			fs.Logf(errString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
+			fs.Logf(nil, "Error %v. Too many requests or write operations. Trying again in %d seconds.", err, e.RateLimitError.RetryAfter)
 			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
 	// Keep old behavior for backward compatibility
+	errString := err.Error()
 	if strings.Contains(errString, "too_many_write_operations") || strings.Contains(errString, "too_many_requests") || errString == "" {
 		return true, err
 	}
@@ -1700,14 +1714,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
-		// If error is insufficient space then don't retry
-		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
-			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
-				err = fserrors.NoRetryError(err)
-				return false, err
-			}
+		if retry, err := shouldRetryExclude(ctx, err); !retry {
+			return retry, err
 		}
-		// after the first chunk is uploaded, we retry everything
+		// after the first chunk is uploaded, we retry everything except the excluded errors
 		return err != nil, err
 	})
 	if err != nil {

backend/googlephotos/googlephotos.go

@@ -1168,7 +1168,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		errors := make([]error, 1)
 		results := make([]*api.MediaItem, 1)
 		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
-		if err != nil {
+		if err == nil {
 			err = errors[0]
 			info = results[0]
 		}

backend/googlephotos/googlephotos_test.go

@@ -2,6 +2,7 @@ package googlephotos
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -35,7 +36,7 @@ func TestIntegration(t *testing.T) {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
-	if err == fs.ErrorNotFoundInConfigFile {
+	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
 		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)

backend/http/http.go

@@ -331,12 +331,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
+	trimmedRemote := strings.TrimLeft(remote, "/") // remove leading "/" since we always have it in f.endpointURL
 	if f.opt.NoEscape {
 		// Directly concatenate without escaping, no_escape behavior
-		return f.endpointURL + remote
+		return f.endpointURL + trimmedRemote
 	}
 	// Default behavior
-	return f.endpointURL + rest.URLPathEscape(remote)
+	return f.endpointURL + rest.URLPathEscape(trimmedRemote)
 }
 
 // Errors returned by parseName

backend/http/http_internal_test.go

@@ -191,6 +191,33 @@ func TestNewObject(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
+func TestNewObjectWithLeadingSlash(t *testing.T) {
+	f := prepare(t)
+
+	o, err := f.NewObject(context.Background(), "/four/under four.txt")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/four/under four.txt", o.Remote())
+	assert.Equal(t, int64(8+lineEndSize), o.Size())
+	_, ok := o.(*Object)
+	assert.True(t, ok)
+
+	// Test the time is correct on the object
+
+	tObj := o.ModTime(context.Background())
+
+	fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+	require.NoError(t, err)
+	tFile := fi.ModTime()
+
+	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+	// check object not found
+	o, err = f.NewObject(context.Background(), "/not found.txt")
+	assert.Nil(t, o)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
+}
+
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

backend/iclouddrive/iclouddrive.go

@@ -445,7 +445,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// build request
-	// cant use normal rename as file needs to be "activated" first
+	// can't use normal rename as file needs to be "activated" first
 
 	r := api.NewUpdateFileInfo()
 	r.DocumentID = doc.DocumentID

backend/imagekit/client/media.go

@@ -75,7 +75,7 @@ type MoveFolderParam struct {
 	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
 }
 
-// JobIDResponse respresents response struct with JobID for folder operations
+// JobIDResponse represents response struct with JobID for folder operations
 type JobIDResponse struct {
 	JobID string `json:"jobId"`
 }

backend/internetarchive/internetarchive.go

@@ -151,6 +151,19 @@ Owner is able to add custom keys. Metadata feature grabs all the keys including
 			Help:     "Host of InternetArchive Frontend.\n\nLeave blank for default value.",
 			Default:  "https://archive.org",
 			Advanced: true,
+		}, {
+			Name: "item_metadata",
+			Help: `Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set.
+Format is key=value and the 'x-archive-meta-' prefix is automatically added.`,
+			Default:  []string{},
+			Hide:     fs.OptionHideConfigurator,
+			Advanced: true,
+		}, {
+			Name: "item_derive",
+			Help: `Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload.
+The derive process produces a number of secondary files from an upload to make an upload more usable on the web.
+Setting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure.`,
+			Default: true,
 		}, {
 			Name: "disable_checksum",
 			Help: `Don't ask the server to test against MD5 checksum calculated by rclone.
@@ -201,6 +214,8 @@ type Options struct {
 	Endpoint        string               `config:"endpoint"`
 	FrontEndpoint   string               `config:"front_endpoint"`
 	DisableChecksum bool                 `config:"disable_checksum"`
+	ItemMetadata    []string             `config:"item_metadata"`
+	ItemDerive      bool                 `config:"item_derive"`
 	WaitArchive     fs.Duration          `config:"wait_archive"`
 	Enc             encoder.MultiEncoder `config:"encoding"`
 }
@@ -790,17 +805,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
-		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
-		"x-archive-auto-make-bucket": "1",    // same as above in IAS3 original way
-		"x-archive-keep-old-version": "0",    // do not keep old versions (a.k.a. trashes in other clouds)
-		"x-archive-meta-mediatype":   "data", // mark media type of the uploading file as "data"
-		"x-archive-queue-derive":     "0",    // skip derivation process (e.g. encoding to smaller files, OCR on PDFs)
-		"x-archive-cascade-delete":   "1",    // enable "cascate delete" (delete all derived files in addition to the file itself)
+		"x-amz-auto-make-bucket":     "1", // create an item if does not exist, do nothing if already
+		"x-archive-auto-make-bucket": "1", // same as above in IAS3 original way
+		"x-archive-keep-old-version": "0", // do not keep old versions (a.k.a. trashes in other clouds)
+		"x-archive-cascade-delete":   "1", // enable "cascate delete" (delete all derived files in addition to the file itself)
 	}
+
 	if size >= 0 {
 		headers["Content-Length"] = fmt.Sprintf("%d", size)
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
+
+	// This is IA's ITEM metadata, not file metadata
+	headers, err = o.appendItemMetadataHeaders(headers, o.fs.opt)
+	if err != nil {
+		return err
+	}
+
 	var mdata fs.Metadata
 	mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
 	if err == nil && mdata != nil {
@@ -863,6 +884,51 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return err
 }
 
+func (o *Object) appendItemMetadataHeaders(headers map[string]string, options Options) (newHeaders map[string]string, err error) {
+	metadataCounter := make(map[string]int)
+	metadataValues := make(map[string][]string)
+
+	// First pass: count occurrences and collect values
+	for _, v := range options.ItemMetadata {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) != 2 {
+			return newHeaders, errors.New("item metadata key=value should be in the form key=value")
+		}
+		key, value := parts[0], parts[1]
+		metadataCounter[key]++
+		metadataValues[key] = append(metadataValues[key], value)
+	}
+
+	// Second pass: add headers with appropriate prefixes
+	for key, count := range metadataCounter {
+		if count == 1 {
+			// Only one occurrence, use x-archive-meta-
+			headers[fmt.Sprintf("x-archive-meta-%s", key)] = metadataValues[key][0]
+		} else {
+			// Multiple occurrences, use x-archive-meta01-, x-archive-meta02-, etc.
+			for i, value := range metadataValues[key] {
+				headers[fmt.Sprintf("x-archive-meta%02d-%s", i+1, key)] = value
+			}
+		}
+	}
+
+	if o.fs.opt.ItemDerive {
+		headers["x-archive-queue-derive"] = "1"
+	} else {
+		headers["x-archive-queue-derive"] = "0"
+	}
+
+	fs.Debugf(o, "Setting IA item derive: %t", o.fs.opt.ItemDerive)
+
+	for k, v := range headers {
+		if strings.HasPrefix(k, "x-archive-meta") {
+			fs.Debugf(o, "Setting IA item metadata: %s=%s", k, v)
+		}
+	}
+
+	return headers, nil
+}
+
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()

backend/local/about_windows.go

@@ -5,18 +5,18 @@ package local
 import (
 	"context"
 	"fmt"
-	"syscall"
 	"unsafe"
 
 	"github.com/rclone/rclone/fs"
+	"golang.org/x/sys/windows"
 )
 
-var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
+var getFreeDiskSpace = windows.NewLazySystemDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var available, total, free int64
-	root, e := syscall.UTF16PtrFromString(f.root)
+	root, e := windows.UTF16PtrFromString(f.root)
 	if e != nil {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e)
 	}
@@ -26,7 +26,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
 		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes
 	)
-	if e1 != syscall.Errno(0) {
+	if e1 != windows.Errno(0) {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e1)
 	}
 	usage := &fs.Usage{

backend/local/local.go

@@ -34,7 +34,6 @@ import (
 // Constants
 const (
 	devUnset   = 0xdeadbeefcafebabe                                     // a device id meaning it is unset
-	linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
 	useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
 )
 
@@ -101,10 +100,8 @@ Metadata is supported on files and directories.
 			},
 			{
 				Name:     "links",
-				Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.",
+				Help:     "Translate symlinks to/from regular files with a '" + fs.LinkSuffix + "' extension for the local backend.",
 				Default:  false,
-				NoPrefix: true,
-				ShortOpt: "l",
 				Advanced: true,
 			},
 			{
@@ -379,17 +376,22 @@ type Directory struct {
 
 var (
 	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
+	errLinksNeedsSuffix  = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
 )
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	ci := fs.GetConfig(ctx)
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
 	}
+	// Override --local-links with --links if set
+	if ci.Links {
+		opt.TranslateSymlinks = true
+	}
 	if opt.TranslateSymlinks && opt.FollowSymlinks {
 		return nil, errLinksAndCopyLinks
 	}
@@ -435,9 +437,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
 	// Check to see if this is a .rclonelink if not found
-	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
+	hasLinkSuffix := strings.HasSuffix(f.root, fs.LinkSuffix)
 	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
-		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
+		fi, err = f.lstat(strings.TrimSuffix(f.root, fs.LinkSuffix))
 	}
 	if err == nil && f.isRegular(fi.Mode()) {
 		// Handle the odd case, that a symlink was specified by name without the link suffix
@@ -508,8 +510,8 @@ func (f *Fs) caseInsensitive() bool {
 //
 // for regular files, localPath is returned unchanged
 func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
+	isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
+	newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
 	return newLocalPath, isTranslatedLink
 }
 
@@ -692,7 +694,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			} else {
 				// Check whether this link should be translated
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += linkSuffix
+					newRemote += fs.LinkSuffix
 				}
 				// Don't include non directory if not included
 				// we leave directory filtering to the layer above

backend/local/local_internal_test.go

@@ -110,7 +110,7 @@ func TestSymlink(t *testing.T) {
 	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
 
 	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
+	file2 := fstest.NewItem("symlink.txt"+fs.LinkSuffix, "file.txt", modTime2)
 
 	// Object viewed as destination
 	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
@@ -139,7 +139,7 @@ func TestSymlink(t *testing.T) {
 
 	// Create a symlink
 	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
+	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+fs.LinkSuffix, "file.txt", modTime3, false)
 	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
 	if haveLChtimes {
 		r.CheckLocalItems(t, file1, file2, file3)
@@ -155,9 +155,9 @@ func TestSymlink(t *testing.T) {
 	assert.Equal(t, "file.txt", linkText)
 
 	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+linkSuffix)
+	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+fs.LinkSuffix)
 	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
+	assert.Equal(t, "symlink2.txt"+fs.LinkSuffix, o.Remote())
 	assert.Equal(t, int64(8), o.Size())
 
 	// Check that NewObject doesn't see the non suffixed version
@@ -165,7 +165,7 @@ func TestSymlink(t *testing.T) {
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
 	// Check that NewFs works with the suffixed version and --links
-	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+fs.LinkSuffix), configmap.Simple{
 		"links": "true",
 	})
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -277,7 +277,7 @@ func TestMetadata(t *testing.T) {
 	// Write a symlink to the file
 	symlinkPath := "metafile-link.txt"
 	osSymlinkPath := filepath.Join(f.root, symlinkPath)
-	symlinkPath += linkSuffix
+	symlinkPath += fs.LinkSuffix
 	require.NoError(t, os.Symlink(filePath, osSymlinkPath))
 	symlinkModTime := fstest.Time("2002-02-03T04:05:10.123123123Z")
 	require.NoError(t, lChtimes(osSymlinkPath, symlinkModTime, symlinkModTime))

backend/onedrive/onedrive.go

@@ -131,7 +131,7 @@ func init() {
 					Help:  "Microsoft Cloud for US Government",
 				}, {
 					Value: regionDE,
-					Help:  "Microsoft Cloud Germany",
+					Help:  "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).",
 				}, {
 					Value: regionCN,
 					Help:  "Azure and Office 365 operated by Vnet Group in China",

backend/opendrive/opendrive.go

@@ -92,6 +92,21 @@ Note that these chunks are buffered in memory so increasing them will
 increase memory use.`,
 			Default:  10 * fs.Mebi,
 			Advanced: true,
+		}, {
+			Name:     "access",
+			Help:     "Files and folders will be uploaded with this access permission (default private)",
+			Default:  "private",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "private",
+				Help:  "The file or folder access can be granted in a way that will allow select users to view, read or write what is absolutely essential for them.",
+			}, {
+				Value: "public",
+				Help:  "The file or folder can be downloaded by anyone from a web browser. The link can be shared in any way,",
+			}, {
+				Value: "hidden",
+				Help:  "The file or folder can be accessed has the same restrictions as  Public if the user knows the URL of the file or folder link in order to access the contents",
+			}},
 		}},
 	})
 }
@@ -102,6 +117,7 @@ type Options struct {
 	Password  string               `config:"password"`
 	Enc       encoder.MultiEncoder `config:"encoding"`
 	ChunkSize fs.SizeSuffix        `config:"chunk_size"`
+	Access    string               `config:"access"`
 }
 
 // Fs represents a remote server
@@ -735,6 +751,23 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
+// getAccessLevel is a helper function to determine access level integer
+func getAccessLevel(access string) int64 {
+	var accessLevel int64
+	switch access {
+	case "private":
+		accessLevel = 0
+	case "public":
+		accessLevel = 1
+	case "hidden":
+		accessLevel = 2
+	default:
+		accessLevel = 0
+		fs.Errorf(nil, "Invalid access: %s, defaulting to private", access)
+	}
+	return accessLevel
+}
+
 // DirCacher methods
 
 // CreateDir makes a directory with pathID as parent and name leaf
@@ -747,7 +780,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			SessionID:           f.session.SessionID,
 			FolderName:          f.opt.Enc.FromStandardName(leaf),
 			FolderSubParent:     pathID,
-			FolderIsPublic:      0,
+			FolderIsPublic:      getAccessLevel(f.opt.Access),
 			FolderPublicUpl:     0,
 			FolderPublicDisplay: 0,
 			FolderPublicDnl:     0,
@@ -1080,7 +1113,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Set permissions
 	err = o.fs.pacer.Call(func() (bool, error) {
-		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: 0}
+		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: getAccessLevel(o.fs.opt.Access)}
 		// fs.Debugf(nil, "Permissions : %#v", update)
 		opts := rest.Opts{
 			Method:     "POST",

backend/oracleobjectstorage/multipart.go

@@ -22,6 +22,7 @@ import (
 	"github.com/oracle/oci-go-sdk/v65/objectstorage"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 )
 
@@ -183,6 +184,9 @@ func (w *objectChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, rea
 			if ossPartNumber <= 8 {
 				return shouldRetry(ctx, resp.HTTPResponse(), err)
 			}
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/pcloud/pcloud.go

@@ -424,7 +424,7 @@ func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) {
 	})
 	// Set our own http client in the context
 	ctx = oauthutil.Context(ctx, baseClient)
-	// create a new oauth client, re-use the token source
+	// create a new oauth client, reuse the token source
 	oAuthClient := oauth2.NewClient(ctx, f.ts)
 	return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil
 }

backend/pikpak/pikpak.go

@@ -213,6 +213,11 @@ Fill in for rclone to use a non root folder as its starting point.
 			Default:  false,
 			Help:     "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
 			Advanced: true,
+		}, {
+			Name:     "no_media_link",
+			Default:  false,
+			Help:     "Use original file links instead of media links.\n\nThis avoids issues caused by invalid media links, but may reduce download speeds.",
+			Advanced: true,
 		}, {
 			Name:     "hash_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate hash if required.",
@@ -286,6 +291,7 @@ type Options struct {
 	RootFolderID        string               `config:"root_folder_id"`
 	UseTrash            bool                 `config:"use_trash"`
 	TrashedOnly         bool                 `config:"trashed_only"`
+	NoMediaLink         bool                 `config:"no_media_link"`
 	HashMemoryThreshold fs.SizeSuffix        `config:"hash_memory_limit"`
 	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
 	UploadConcurrency   int                  `config:"upload_concurrency"`
@@ -1575,15 +1581,14 @@ func (o *Object) setMetaData(info *api.File) (err error) {
 	o.md5sum = info.Md5Checksum
 	if info.Links.ApplicationOctetStream != nil {
 		o.link = info.Links.ApplicationOctetStream
-		if fid := parseFileID(o.link.URL); fid != "" {
-			for mid, media := range info.Medias {
-				if media.Link == nil {
-					continue
-				}
-				if mfid := parseFileID(media.Link.URL); fid == mfid {
-					fs.Debugf(o, "Using a media link from Medias[%d]", mid)
-					o.link = media.Link
-					break
+		if !o.fs.opt.NoMediaLink {
+			if fid := parseFileID(o.link.URL); fid != "" {
+				for _, media := range info.Medias {
+					if media.Link != nil && parseFileID(media.Link.URL) == fid {
+						fs.Debugf(o, "Using a media link")
+						o.link = media.Link
+						break
+					}
 				}
 			}
 		}

backend/s3/ibm_signer.go

@@ -0,0 +1,59 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+)
+
+// Authenticator defines an interface for obtaining an IAM token.
+type Authenticator interface {
+	GetToken() (string, error)
+}
+
+// IbmIamSigner is a structure for signing requests using IBM IAM.
+// Requeres APIKey and Resource InstanceID
+type IbmIamSigner struct {
+	APIKey     string
+	InstanceID string
+	Auth       Authenticator
+}
+
+// SignHTTP signs requests using IBM IAM token.
+func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error {
+	var authenticator Authenticator
+	if signer.Auth != nil {
+		authenticator = signer.Auth
+	} else {
+		authenticator = &core.IamAuthenticator{ApiKey: signer.APIKey}
+	}
+	token, err := authenticator.GetToken()
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("ibm-service-instance-id", signer.InstanceID)
+	return nil
+}
+
+// NoOpCredentialsProvider is needed since S3 SDK requires having credentials, eventhough authentication is happening via IBM IAM.
+type NoOpCredentialsProvider struct{}
+
+// Retrieve returns mock credentials for the NoOpCredentialsProvider.
+func (n *NoOpCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID:     "NoOpAccessKey",
+		SecretAccessKey: "NoOpSecretKey",
+		SessionToken:    "",
+		Source:          "NoOpCredentialsProvider",
+	}, nil
+}
+
+// IsExpired always returns false
+func (n *NoOpCredentialsProvider) IsExpired() bool {
+	return false
+}

backend/s3/ibm_signer_test.go

@@ -0,0 +1,47 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/stretchr/testify/assert"
+)
+
+type MockAuthenticator struct {
+	Token string
+	Error error
+}
+
+func (m *MockAuthenticator) GetToken() (string, error) {
+	return m.Token, m.Error
+}
+
+func TestSignHTTP(t *testing.T) {
+	apiKey := "mock-api-key"
+	instanceID := "mock-instance-id"
+	token := "mock-iam-token"
+	mockAuth := &MockAuthenticator{
+		Token: token,
+		Error: nil,
+	}
+	signer := &IbmIamSigner{
+		APIKey:     apiKey,
+		InstanceID: instanceID,
+		Auth:       mockAuth,
+	}
+	req, err := http.NewRequest("GET", "https://example.com", nil)
+	if err != nil {
+		t.Fatalf("Failed to create HTTP request: %v", err)
+	}
+	credentials := aws.Credentials{
+		AccessKeyID:     "mock-access-key",
+		SecretAccessKey: "mock-secret-key",
+	}
+	err = signer.SignHTTP(context.TODO(), credentials, req, "payload-hash", "service", "region", time.Now())
+	assert.NoError(t, err, "Expected no error")
+	assert.Equal(t, "Bearer "+token, req.Header.Get("Authorization"), "Authorization header should be set correctly")
+	assert.Equal(t, instanceID, req.Header.Get("ibm-service-instance-id"), "ibm-service-instance-id header should be set correctly")
+}

backend/s3/s3.go

@@ -36,8 +36,8 @@ import (
 	"github.com/aws/smithy-go/logging"
 	"github.com/aws/smithy-go/middleware"
 	smithyhttp "github.com/aws/smithy-go/transport/http"
-
 	"github.com/ncw/swift/v2"
+
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -1343,7 +1343,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1356,6 +1356,10 @@ func init() {
 				Value:    "sfo3.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces San Francisco 3",
 				Provider: "DigitalOcean",
+			}, {
+				Value:    "sfo2.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces San Francisco 2",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "fra1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Frankfurt 1",
@@ -1372,6 +1376,18 @@ func init() {
 				Value:    "sgp1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
+			}, {
+				Value:    "lon1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces London 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "tor1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Toronto 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "blr1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Bangalore 1",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
@@ -1476,14 +1492,6 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
-			}, {
-				Value:    "br-se1.magaluobjects.com",
-				Help:     "Magalu BR Southeast 1 endpoint",
-				Provider: "Magalu",
-			}, {
-				Value:    "br-ne1.magaluobjects.com",
-				Help:     "Magalu BR Northeast 1 endpoint",
-				Provider: "Magalu",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -2056,7 +2064,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "One Zone Infrequent Access storage class",
 			}, {
 				Value: "GLACIER",
-				Help:  "Glacier storage class",
+				Help:  "Glacier Flexible Retrieval storage class",
 			}, {
 				Value: "DEEP_ARCHIVE",
 				Help:  "Glacier Deep Archive storage class",
@@ -2122,13 +2130,16 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "Standard storage class",
 			}},
 		}, {
-			// Mapping from here: #todo
+			// Mapping from here: https://docs.magalu.cloud/docs/storage/object-storage/Classes-de-Armazenamento/standard
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in Magalu.",
 			Provider: "Magalu",
 			Examples: []fs.OptionExample{{
 				Value: "STANDARD",
 				Help:  "Standard storage class",
+			}, {
+				Value: "GLACIER_IR",
+				Help:  "Glacier Instant Retrieval storage class",
 			}},
 		}, {
 			// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
@@ -2669,6 +2680,34 @@ knows about - please make a bug report if not.
 You can change this if you want to disable the use of multipart uploads.
 This shouldn't be necessary in normal operation.
 
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "use_x_id",
+			Help: `Set if rclone should add x-id URL parameters.
+
+You can change this if you want to disable the AWS SDK from
+adding x-id URL parameters.
+
+This shouldn't be necessary in normal operation.
+
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "sign_accept_encoding",
+			Help: `Set if rclone should include Accept-Encoding as part of the signature.
+
+You can change this if you want to stop rclone including
+Accept-Encoding as part of the signature.
+
+This shouldn't be necessary in normal operation.
+
 This should be automatically set correctly for all providers rclone
 knows about - please make a bug report if not.
 `,
@@ -2725,6 +2764,16 @@ use |-vv| to see the debug level logs.
 			Default:  sdkLogMode(0),
 			Advanced: true,
 		},
+			{
+				Name:     "ibm_api_key",
+				Help:     "IBM API Key to be used to obtain IAM token",
+				Provider: "IBMCOS",
+			},
+			{
+				Name:     "ibm_resource_instance_id",
+				Help:     "IBM service instance id",
+				Provider: "IBMCOS",
+			},
 		}})
 }
 
@@ -2878,6 +2927,10 @@ type Options struct {
 	UseUnsignedPayload    fs.Tristate          `config:"use_unsigned_payload"`
 	SDKLogMode            sdkLogMode           `config:"sdk_log_mode"`
 	DirectoryBucket       bool                 `config:"directory_bucket"`
+	IBMAPIKey             string               `config:"ibm_api_key"`
+	IBMInstanceID         string               `config:"ibm_resource_instance_id"`
+	UseXID                fs.Tristate          `config:"use_x_id"`
+	SignAcceptEncoding    fs.Tristate          `config:"sign_accept_encoding"`
 }
 
 // Fs represents a remote s3 server
@@ -3062,59 +3115,67 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 	}
 }
 
+// Fixup the request if needed.
+//
 // Google Cloud Storage alters the Accept-Encoding header, which
-// breaks the v2 request signature
+// breaks the v2 request signature. This is set with opt.SignAcceptEncoding.
 //
 // It also doesn't like the x-id URL parameter SDKv2 puts in so we
-// remove that too.
+// remove that too. This is set with opt.UseXID.Value.
 //
 // See https://github.com/aws/aws-sdk-go-v2/issues/1816.
 // Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
-func fixupGCS(o *s3.Options) {
+func fixupRequest(o *s3.Options, opt *Options) {
 	type ignoredHeadersKey struct{}
 	headers := []string{"Accept-Encoding"}
 
 	fixup := middleware.FinalizeMiddlewareFunc(
-		"FixupGCS",
+		"FixupRequest",
 		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
 			req, ok := in.Request.(*smithyhttp.Request)
 			if !ok {
-				return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+				return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
 			}
 
-			// Delete headers from being signed - will restore later
-			ignored := make(map[string]string, len(headers))
-			for _, h := range headers {
-				ignored[h] = req.Header.Get(h)
-				req.Header.Del(h)
+			if !opt.SignAcceptEncoding.Value {
+				// Delete headers from being signed - will restore later
+				ignored := make(map[string]string, len(headers))
+				for _, h := range headers {
+					ignored[h] = req.Header.Get(h)
+					req.Header.Del(h)
+				}
+
+				// Store ignored on context
+				ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
 			}
 
-			// Remove x-id because Google doesn't like them
-			if query := req.URL.Query(); query.Has("x-id") {
-				query.Del("x-id")
-				req.URL.RawQuery = query.Encode()
+			if !opt.UseXID.Value {
+				// Remove x-id
+				if query := req.URL.Query(); query.Has("x-id") {
+					query.Del("x-id")
+					req.URL.RawQuery = query.Encode()
+				}
 			}
 
-			// Store ignored on context
-			ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
-
 			return next.HandleFinalize(ctx, in)
 		},
 	)
 
 	// Restore headers if necessary
 	restore := middleware.FinalizeMiddlewareFunc(
-		"FixupGCSRestoreHeaders",
+		"FixupRequestRestoreHeaders",
 		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
 			req, ok := in.Request.(*smithyhttp.Request)
 			if !ok {
-				return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+				return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
 			}
 
-			// Restore ignored from ctx
-			ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
-			for k, v := range ignored {
-				req.Header.Set(k, v)
+			if !opt.SignAcceptEncoding.Value {
+				// Restore ignored from ctx
+				ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
+				for k, v := range ignored {
+					req.Header.Set(k, v)
+				}
 			}
 
 			return next.HandleFinalize(ctx, in)
@@ -3160,6 +3221,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 
 	// Try to fill in the config from the environment if env_auth=true
 	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
+
 		configOpts := []func(*awsconfig.LoadOptions) error{}
 		// Set the name of the profile if supplied
 		if opt.Profile != "" {
@@ -3173,8 +3235,12 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 		if err != nil {
 			return nil, fmt.Errorf("couldn't load configuration with env_auth=true: %w", err)
 		}
+
 	} else {
 		switch {
+		case opt.Provider == "IBMCOS" && opt.V2Auth:
+			awsConfig.Credentials = &NoOpCredentialsProvider{}
+			fs.Debugf(nil, "Using IBM IAM")
 		case opt.AccessKeyID == "" && opt.SecretAccessKey == "":
 			// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
 			awsConfig.Credentials = aws.AnonymousCredentials{}
@@ -3228,14 +3294,21 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 
 	if opt.V2Auth || opt.Region == "other-v2-signature" {
 		fs.Debugf(nil, "Using v2 auth")
-		options = append(options, func(s3Opt *s3.Options) {
-			s3Opt.HTTPSignerV4 = &v2Signer{opt: opt}
-		})
+		if opt.Provider == "IBMCOS" && opt.IBMAPIKey != "" && opt.IBMInstanceID != "" {
+			options = append(options, func(s3Opt *s3.Options) {
+				s3Opt.HTTPSignerV4 = &IbmIamSigner{APIKey: opt.IBMAPIKey, InstanceID: opt.IBMInstanceID}
+			})
+		} else {
+			options = append(options, func(s3Opt *s3.Options) {
+				s3Opt.HTTPSignerV4 = &v2Signer{opt: opt}
+			})
+		}
 	}
 
-	if opt.Provider == "GCS" {
+	// Fixup the request if needed
+	if !opt.UseXID.Value || !opt.SignAcceptEncoding.Value {
 		options = append(options, func(o *s3.Options) {
-			fixupGCS(o)
+			fixupRequest(o, opt)
 		})
 	}
 
@@ -3344,12 +3417,14 @@ func setQuirks(opt *Options) {
 		listObjectsV2         = true // Always use ListObjectsV2 instead of ListObjects
 		virtualHostStyle      = true // Use bucket.provider.com instead of putting the bucket in the URL
 		urlEncodeListings     = true // URL encode the listings to help with control characters
-		useMultipartEtag      = true // Set if Etags for multpart uploads are compatible with AWS
+		useMultipartEtag      = true // Set if Etags for multipart uploads are compatible with AWS
 		useAcceptEncodingGzip = true // Set Accept-Encoding: gzip
 		mightGzip             = true // assume all providers might use content encoding gzip until proven otherwise
 		useAlreadyExists      = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
 		useMultipartUploads   = true // Set if provider supports multipart uploads
 		useUnsignedPayload    = true // Do we need to use unsigned payloads to avoid seeking in PutObject
+		useXID                = true // Add x-id URL parameter into requests
+		signAcceptEncoding    = true // If we should include AcceptEncoding in the signature
 	)
 	switch opt.Provider {
 	case "AWS":
@@ -3494,11 +3569,14 @@ func setQuirks(opt *Options) {
 		// Google break request Signature by mutating accept-encoding HTTP header
 		// https://github.com/rclone/rclone/issues/6670
 		useAcceptEncodingGzip = false
+		signAcceptEncoding = false
 		useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
 		// GCS S3 doesn't support multi-part server side copy:
 		// See: https://issuetracker.google.com/issues/323465186
 		// So make cutoff very large which it does seem to support
 		opt.CopyCutoff = math.MaxInt64
+		// GCS doesn't like the x-id URL parameter the SDKv2 inserts
+		useXID = false
 	default: //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last case
 		fs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)
 		fallthrough
@@ -3568,6 +3646,18 @@ func setQuirks(opt *Options) {
 		opt.UseUnsignedPayload.Valid = true
 		opt.UseUnsignedPayload.Value = useUnsignedPayload
 	}
+
+	// Set the correct use UseXID if not manually set
+	if !opt.UseXID.Valid {
+		opt.UseXID.Valid = true
+		opt.UseXID.Value = useXID
+	}
+
+	// Set the correct SignAcceptEncoding if not manually set
+	if !opt.SignAcceptEncoding.Valid {
+		opt.SignAcceptEncoding.Valid = true
+		opt.SignAcceptEncoding.Value = signAcceptEncoding
+	}
 }
 
 // setRoot changes the root of the Fs
@@ -3682,6 +3772,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.Provider == "IDrive" {
 		f.features.SetTier = false
 	}
+	if opt.Provider == "AWS" {
+		f.features.DoubleSlash = true
+	}
 	if opt.DirectoryMarkers {
 		f.features.CanHaveEmptyDirectories = true
 	}
@@ -4153,7 +4246,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 		opt.prefix += "/"
 	}
 	if !opt.findFile {
-		if opt.directory != "" {
+		if opt.directory != "" && (opt.prefix == "" && !bucket.IsAllSlashes(opt.directory) || opt.prefix != "" && !strings.HasSuffix(opt.directory, "/")) {
 			opt.directory += "/"
 		}
 	}
@@ -4250,14 +4343,18 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 				}
 				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, opt.prefix) {
-					fs.Logf(f, "Odd name received %q", remote)
+					fs.Logf(f, "Odd directory name received %q", remote)
 					continue
 				}
 				remote = remote[len(opt.prefix):]
+				// Trim one slash off the remote name
+				remote, _ = strings.CutSuffix(remote, "/")
+				if remote == "" || bucket.IsAllSlashes(remote) {
+					remote += "/"
+				}
 				if opt.addBucket {
 					remote = bucket.Join(opt.bucket, remote)
 				}
-				remote = strings.TrimSuffix(remote, "/")
 				err = fn(remote, &types.Object{Key: &remote}, nil, true)
 				if err != nil {
 					if err == errEndList {
@@ -6057,7 +6154,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 			if mOut == nil {
 				err = fserrors.RetryErrorf("internal error: no info from multipart upload")
 			} else if mOut.UploadId == nil {
-				err = fserrors.RetryErrorf("internal error: no UploadId in multpart upload: %#v", *mOut)
+				err = fserrors.RetryErrorf("internal error: no UploadId in multipart upload: %#v", *mOut)
 			}
 		}
 		return f.shouldRetry(ctx, err)
@@ -6175,6 +6272,9 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/smb/connpool.go

@@ -2,8 +2,10 @@ package smb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"os"
 	"time"
 
 	smb2 "github.com/cloudsoda/go-smb2"
@@ -11,14 +13,17 @@ import (
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fshttp"
+	"golang.org/x/sync/errgroup"
 )
 
 // dial starts a client connection to the given SMB server. It is a
 // convenience function that connects to the given network address,
 // initiates the SMB handshake, and then sets up a Client.
+//
+// The context is only used for establishing the connection, not after.
 func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 	dialer := fshttp.NewDialer(ctx)
-	tconn, err := dialer.Dial(network, addr)
+	tconn, err := dialer.DialContext(ctx, network, addr)
 	if err != nil {
 		return nil, err
 	}
@@ -31,13 +36,29 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 		}
 	}
 
-	d := &smb2.Dialer{
-		Initiator: &smb2.NTLMInitiator{
+	d := &smb2.Dialer{}
+	if f.opt.UseKerberos {
+		cl, err := getKerberosClient()
+		if err != nil {
+			return nil, err
+		}
+
+		spn := f.opt.SPN
+		if spn == "" {
+			spn = "cifs/" + f.opt.Host
+		}
+
+		d.Initiator = &smb2.Krb5Initiator{
+			Client:    cl,
+			TargetSPN: spn,
+		}
+	} else {
+		d.Initiator = &smb2.NTLMInitiator{
 			User:      f.opt.User,
 			Password:  pass,
 			Domain:    f.opt.Domain,
 			TargetSPN: f.opt.SPN,
-		},
+		}
 	}
 
 	session, err := d.DialConn(ctx, tconn, addr)
@@ -73,15 +94,7 @@ func (c *conn) close() (err error) {
 
 // True if it's closed
 func (c *conn) closed() bool {
-	var nopErr error
-	if c.smbShare != nil {
-		// stat the current directory
-		_, nopErr = c.smbShare.Stat(".")
-	} else {
-		// list the shares
-		_, nopErr = c.smbSession.ListSharenames()
-	}
-	return nopErr != nil
+	return c.smbSession.Echo() != nil
 }
 
 // Show that we are using a SMB session
@@ -102,23 +115,20 @@ func (f *Fs) getSessions() int32 {
 }
 
 // Open a new connection to the SMB server.
+//
+// The context is only used for establishing the connection, not after.
 func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) {
-	// As we are pooling these connections we need to decouple
-	// them from the current context
-	bgCtx := context.Background()
-
-	c, err = f.dial(bgCtx, "tcp", f.opt.Host+":"+f.opt.Port)
+	c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SMB: %w", err)
 	}
 	if share != "" {
 		// mount the specified share as well if user requested
-		c.smbShare, err = c.smbSession.Mount(share)
+		err = c.mountShare(share)
 		if err != nil {
 			_ = c.smbSession.Logoff()
 			return nil, fmt.Errorf("couldn't initialize SMB: %w", err)
 		}
-		c.smbShare = c.smbShare.WithContext(bgCtx)
 	}
 	return c, nil
 }
@@ -176,23 +186,30 @@ func (f *Fs) getConnection(ctx context.Context, share string) (c *conn, err erro
 // Return a SMB connection to the pool
 //
 // It nils the pointed to connection out so it can't be reused
-func (f *Fs) putConnection(pc **conn) {
-	c := *pc
-	*pc = nil
-
-	var nopErr error
-	if c.smbShare != nil {
-		// stat the current directory
-		_, nopErr = c.smbShare.Stat(".")
-	} else {
-		// list the shares
-		_, nopErr = c.smbSession.ListSharenames()
-	}
-	if nopErr != nil {
-		fs.Debugf(f, "Connection failed, closing: %v", nopErr)
-		_ = c.close()
+//
+// if err is not nil then it checks the connection is alive using an
+// ECHO request
+func (f *Fs) putConnection(pc **conn, err error) {
+	if pc == nil {
 		return
 	}
+	c := *pc
+	if c == nil {
+		return
+	}
+	*pc = nil
+	if err != nil {
+		// If not a regular SMB error then check the connection
+		if !(errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrExist) || errors.Is(err, os.ErrPermission)) {
+			echoErr := c.smbSession.Echo()
+			if echoErr != nil {
+				fs.Debugf(f, "Connection failed, closing: %v", echoErr)
+				_ = c.close()
+				return
+			}
+			fs.Debugf(f, "Connection OK after error: %v", err)
+		}
+	}
 
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
@@ -219,15 +236,18 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
 	if len(f.pool) != 0 {
 		fs.Debugf(f, "Closing %d unused connections", len(f.pool))
 	}
+
+	g, _ := errgroup.WithContext(ctx)
 	for i, c := range f.pool {
-		if !c.closed() {
-			cErr := c.close()
-			if cErr != nil {
-				err = cErr
+		g.Go(func() (err error) {
+			if !c.closed() {
+				err = c.close()
 			}
-		}
-		f.pool[i] = nil
+			f.pool[i] = nil
+			return err
+		})
 	}
+	err = g.Wait()
 	f.pool = nil
 	return err
 }

backend/smb/kerberos.go

@@ -0,0 +1,78 @@
+package smb
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/jcmturner/gokrb5/v8/client"
+	"github.com/jcmturner/gokrb5/v8/config"
+	"github.com/jcmturner/gokrb5/v8/credentials"
+)
+
+var (
+	kerberosClient *client.Client
+	kerberosErr    error
+	kerberosOnce   sync.Once
+)
+
+// getKerberosClient returns a Kerberos client that can be used to authenticate.
+func getKerberosClient() (*client.Client, error) {
+	if kerberosClient == nil || kerberosErr == nil {
+		kerberosOnce.Do(func() {
+			kerberosClient, kerberosErr = createKerberosClient()
+		})
+	}
+
+	return kerberosClient, kerberosErr
+}
+
+// createKerberosClient creates a new Kerberos client.
+func createKerberosClient() (*client.Client, error) {
+	cfgPath := os.Getenv("KRB5_CONFIG")
+	if cfgPath == "" {
+		cfgPath = "/etc/krb5.conf"
+	}
+
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine the ccache location from the environment, falling back to the
+	// default location.
+	ccachePath := os.Getenv("KRB5CCNAME")
+	switch {
+	case strings.Contains(ccachePath, ":"):
+		parts := strings.SplitN(ccachePath, ":", 2)
+		switch parts[0] {
+		case "FILE":
+			ccachePath = parts[1]
+		case "DIR":
+			primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
+			if err != nil {
+				return nil, err
+			}
+			ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
+		default:
+			return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
+		}
+	case ccachePath == "":
+		u, err := user.Current()
+		if err != nil {
+			return nil, err
+		}
+
+		ccachePath = "/tmp/krb5cc_" + u.Uid
+	}
+
+	ccache, err := credentials.LoadCCache(ccachePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.NewFromCCache(ccache, cfg)
+}

backend/smb/smb.go

@@ -25,7 +25,7 @@ import (
 )
 
 const (
-	minSleep      = 100 * time.Millisecond
+	minSleep      = 10 * time.Millisecond
 	maxSleep      = 2 * time.Second
 	decayConstant = 2 // bigger for slower decay, exponential
 )
@@ -76,6 +76,16 @@ authentication, and it often needs to be set for clusters. For example:
 Leave blank if not sure.
 `,
 			Sensitive: true,
+		}, {
+			Name: "use_kerberos",
+			Help: `Use Kerberos authentication.
+
+If set, rclone will use Kerberos authentication instead of NTLM. This
+requires a valid Kerberos configuration and credentials cache to be
+available, either in the default locations or as specified by the
+KRB5_CONFIG and KRB5CCNAME environment variables.
+`,
+			Default: false,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -126,6 +136,7 @@ type Options struct {
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
 	SPN             string      `config:"spn"`
+	UseKerberos     bool        `config:"use_kerberos"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`
@@ -196,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 	stat, err := cn.smbShare.Stat(f.toSambaPath(dir))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		// ignore stat error here
 		return f, nil
@@ -257,7 +268,7 @@ func (f *Fs) findObjectSeparate(ctx context.Context, share, path string) (fs.Obj
 		return nil, err
 	}
 	stat, err := cn.smbShare.Stat(f.toSambaPath(path))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, translateError(err, false)
 	}
@@ -279,7 +290,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 		return err
 	}
 	err = cn.smbShare.MkdirAll(f.toSambaPath(path), 0o755)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -294,7 +305,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return err
 	}
 	err = cn.smbShare.Remove(f.toSambaPath(path))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -364,7 +375,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		return nil, err
 	}
 	err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, translateError(err, false)
 	}
@@ -401,7 +412,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	if err != nil {
 		return err
 	}
-	defer f.putConnection(&cn)
+	defer f.putConnection(&cn, err)
 
 	_, err = cn.smbShare.Stat(dstPath)
 	if os.IsNotExist(err) {
@@ -419,7 +430,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if err != nil {
 		return nil, err
 	}
-	defer f.putConnection(&cn)
+	defer f.putConnection(&cn, err)
 
 	if share == "" {
 		shares, err := cn.smbSession.ListSharenames()
@@ -463,7 +474,7 @@ func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) {
 		return nil, err
 	}
 	stat, err := cn.smbShare.Statfs(dir)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, err
 	}
@@ -545,7 +556,7 @@ func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error {
 		return err
 	}
 	err = cn.smbShare.MkdirAll(f.toSambaPath(dir), 0o755)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -593,7 +604,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	if err != nil {
 		return err
 	}
-	defer o.fs.putConnection(&cn)
+	defer o.fs.putConnection(&cn, err)
 
 	err = cn.smbShare.Chtimes(reqDir, t, t)
 	if err != nil {
@@ -601,9 +612,10 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	}
 
 	fi, err := cn.smbShare.Stat(reqDir)
-	if err == nil {
-		o.statResult = fi
+	if err != nil {
+		return fmt.Errorf("SetModTime: stat: %w", err)
 	}
+	o.statResult = fi
 	return err
 }
 
@@ -638,24 +650,25 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	fl, err := cn.smbShare.OpenFile(filename, os.O_RDONLY, 0)
 	if err != nil {
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 		return nil, fmt.Errorf("failed to open: %w", err)
 	}
 	pos, err := fl.Seek(offset, io.SeekStart)
 	if err != nil {
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 		return nil, fmt.Errorf("failed to seek: %w", err)
 	}
 	if pos != offset {
-		o.fs.putConnection(&cn)
-		return nil, fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos)
+		err = fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos)
+		o.fs.putConnection(&cn, err)
+		return nil, err
 	}
 
 	in = readers.NewLimitedReadCloser(fl, limit)
 	in = &boundReadCloser{
 		rc: in,
 		close: func() error {
-			o.fs.putConnection(&cn)
+			o.fs.putConnection(&cn, nil)
 			return nil
 		},
 	}
@@ -685,8 +698,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 	defer func() {
-		o.statResult, _ = cn.smbShare.Stat(filename)
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 	}()
 
 	fl, err := cn.smbShare.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
@@ -723,7 +735,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update Close failed: %w", err)
 	}
 
-	// Set the modified time
+	// Set the modified time and also o.statResult
 	err = o.SetModTime(ctx, src.ModTime(ctx))
 	if err != nil {
 		return fmt.Errorf("Update SetModTime failed: %w", err)
@@ -746,7 +758,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	}
 
 	err = cn.smbShare.Remove(filename)
-	o.fs.putConnection(&cn)
+	o.fs.putConnection(&cn, err)
 
 	return err
 }

backend/smb/smb_test.go

@@ -2,6 +2,7 @@
 package smb_test
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/rclone/rclone/backend/smb"
@@ -15,3 +16,13 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*smb.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	krb5Dir := t.TempDir()
+	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
+	t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSMBKerberos:rclone",
+		NilObject:  (*smb.Object)(nil),
+	})
+}

backend/sugarsync/sugarsync.go

@@ -120,7 +120,7 @@ func init() {
 				srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) //  FIXME
 
 				// FIXME
-				//err = f.pacer.Call(func() (bool, error) {
+				// err = f.pacer.Call(func() (bool, error) {
 				resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
 				//	return shouldRetry(ctx, resp, err)
 				//})
@@ -327,7 +327,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
 func (f *Fs) getAuthToken(ctx context.Context) error {
 	fs.Debugf(f, "Renewing token")
 
-	var authRequest = api.TokenAuthRequest{
+	authRequest := api.TokenAuthRequest{
 		AccessKeyID:      withDefault(f.opt.AccessKeyID, accessKeyID),
 		PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)),
 		RefreshToken:     f.opt.RefreshToken,
@@ -509,7 +509,7 @@ func errorHandler(resp *http.Response) (err error) {
 		return fmt.Errorf("error reading error out of body: %w", err)
 	}
 	match := findError.FindSubmatch(body)
-	if match == nil || len(match) < 2 || len(match[1]) == 0 {
+	if len(match) < 2 || len(match[1]) == 0 {
 		return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
 	}
 	return fmt.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1])
@@ -552,7 +552,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
-	//fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
+	// fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
 	// Find the leaf in pathID
 	found, err = f.listAll(ctx, pathID, nil, func(item *api.Collection) bool {
 		if strings.EqualFold(item.Name, leaf) {

backend/webdav/webdav.go

@@ -161,7 +161,24 @@ Set to 0 to disable chunked uploading.
 			Default:  false,
 		},
 			fshttp.UnixSocketConfig,
-		},
+			{
+				Name: "auth_redirect",
+				Help: `Preserve authentication on redirect.
+
+If the server redirects rclone to a new domain when it is trying to
+read a file then normally rclone will drop the Authorization: header
+from the request.
+
+This is standard security practice to avoid sending your credentials
+to an unknown webserver.
+
+However this is desirable in some circumstances. If you are getting
+an error like "401 Unauthorized" when rclone is attempting to read
+files from the webdav server then you can try this option.
+`,
+				Advanced: true,
+				Default:  false,
+			}},
 	})
 }
 
@@ -180,6 +197,7 @@ type Options struct {
 	ExcludeShares      bool                 `config:"owncloud_exclude_shares"`
 	ExcludeMounts      bool                 `config:"owncloud_exclude_mounts"`
 	UnixSocket         string               `config:"unix_socket"`
+	AuthRedirect       bool                 `config:"auth_redirect"`
 }
 
 // Fs represents a remote webdav
@@ -1456,6 +1474,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		ExtraHeaders: map[string]string{
 			"Depth": "0",
 		},
+		AuthRedirect: o.fs.opt.AuthRedirect, // allow redirects to preserve Auth
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)

bin/make_manual.py

@@ -7,6 +7,7 @@ conversion into man pages etc.
 import os
 import re
 import time
+import subprocess
 from datetime import datetime
 
 docpath = "docs/content"
@@ -35,6 +36,7 @@ docs = [
     "box.md",
     "cache.md",
     "chunker.md",
+    "cloudinary.md",
     "sharefile.md",
     "crypt.md",
     "compress.md",
@@ -191,13 +193,23 @@ def main():
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
     build_date = datetime.utcfromtimestamp(
             int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
+    help_output = subprocess.check_output(["rclone", "help"]).decode("utf-8")
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-""" % build_date.strftime("%b %d, %Y"))
+# NAME
+
+rclone - manage files on cloud storage
+
+# SYNOPSIS
+
+```
+%s
+```
+""" % (build_date.strftime("%b %d, %Y"), help_output))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

bin/not-in-stable.go

@@ -29,7 +29,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 	cmd := exec.Command("git", "log", "--oneline", from+".."+to)
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
+		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
 	}
 	logMap = map[string]string{}
 	logs = []string{}
@@ -39,7 +39,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 		}
 		match := logRe.FindSubmatch(line)
 		if match == nil {
-			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
+			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
 		}
 		var hash, logMessage = string(match[1]), string(match[2])
 		logMap[logMessage] = hash
@@ -52,12 +52,12 @@ func main() {
 	flag.Parse()
 	args := flag.Args()
 	if len(args) != 0 {
-		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
+		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
 	}
 	// v1.54.0
 	versionBytes, err := os.ReadFile("VERSION")
 	if err != nil {
-		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
+		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
 	}
 	if versionBytes[0] == 'v' {
 		versionBytes = versionBytes[1:]
@@ -65,7 +65,7 @@ func main() {
 	versionBytes = bytes.TrimSpace(versionBytes)
 	semver := semver.New(string(versionBytes))
 	stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1)
-	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
+	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
 	masterMap, masterLogs := readCommits(stable+".0", "master")
 	stableMap, _ := readCommits(stable+".0", stable+"-stable")
 	for _, logMessage := range masterLogs {

bin/test_backend_sizes.py

@@ -7,15 +7,18 @@ Run with no arguments to test all backends or a supply a list of
 backends to test.
 """
 
+import os
+import re
+import sys
+import subprocess
+
 all_backends = "backend/all/all.go"
 
 # compile command which is more or less like the production builds
 compile_command = ["go", "build", "--ldflags", "-s", "-trimpath"]
 
-import os
-import re
-import sys
-import subprocess
+# disable CGO as that makes a lot of difference to binary size
+os.environ["CGO_ENABLED"]="0"
 
 match_backend = re.compile(r'"github.com/rclone/rclone/backend/(.*?)"')
 
@@ -43,6 +46,9 @@ def write_all(orig_all, backend):
             # Comment out line matching backend
             if match and match.group(1) == backend:
                 line = "// " + line
+            # s3 and pikpak depend on each other
+            if backend == "s3" and "pikpak" in line:
+                line = "// " + line
             fd.write(line+"\n")
 
 def compile():

cmd/bisync/bisync_debug_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/rclone/rclone/cmd/bisync/bilib"
 	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 const configFile = "../../fstest/test_all/config.yaml"

cmd/bisync/bisync_test.go

@@ -108,6 +108,8 @@ var logReplacements = []string{
 	`^.*?Can't compare hashes, so using check --download.*?$`, dropMe,
 	// ignore timestamps in directory time updates
 	`^(INFO  : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe,
+	// ignore equivalent log for backends lacking dir modtime support
+	`^(INFO  : .*?: Making directory).*$`, dropMe,
 	// ignore sizes in directory time updates
 	`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe,
 	// ignore sizes in directory metadata updates
@@ -746,6 +748,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 	case "test-func":
 		b.TestFn = testFunc
 		return
+	case "concurrent-func":
+		b.TestFn = func() {
+			src := filepath.Join(b.dataDir, "file7.txt")
+			dst := "file1.txt"
+			err := b.copyFile(ctx, src, b.replaceHex(b.path2), dst)
+			if err != nil {
+				fs.Errorf(src, "error copying file: %v", err)
+			}
+		}
+		return
 	case "fix-names":
 		// in case the local os converted any filenames
 		ci.NoUnicodeNormalization = true
@@ -871,10 +883,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 		if !ok || err != nil {
 			fs.Logf(remotePath, "Can't find expected file %s (was it renamed by the os?) %v", args[1], err)
 			return
-		} else {
-			// include hash of filename to make unicode form differences easier to see in logs
-			fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf))
 		}
+		// include hash of filename to make unicode form differences easier to see in logs
+		fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf))
 		return
 	default:
 		return fmt.Errorf("unknown command: %q", args[0])

cmd/bisync/compare.go

@@ -63,40 +63,40 @@ func (b *bisyncRun) setCompareDefaults(ctx context.Context) error {
 	}
 
 	if b.opt.Compare.SlowHashSyncOnly && b.opt.Compare.SlowHashDetected && b.opt.Resync {
-		fs.Logf(nil, Color(terminal.Dim, "Ignoring checksums during --resync as --slow-hash-sync-only is set.")) ///nolint:govet
+		fs.Log(nil, Color(terminal.Dim, "Ignoring checksums during --resync as --slow-hash-sync-only is set."))
 		ci.CheckSum = false
 		// note not setting b.opt.Compare.Checksum = false as we still want to build listings on the non-slow side, if any
 	} else if b.opt.Compare.Checksum && !ci.CheckSum {
-		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Checksums will be compared for deltas but not during sync as --checksum is not set.")) //nolint:govet
+		fs.Log(nil, Color(terminal.YellowFg, "WARNING: Checksums will be compared for deltas but not during sync as --checksum is not set."))
 	}
 	if b.opt.Compare.Modtime && (b.fs1.Precision() == fs.ModTimeNotSupported || b.fs2.Precision() == fs.ModTimeNotSupported) {
-		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Modtime compare was requested but at least one remote does not support it. It is recommended to use --checksum or --size-only instead.")) //nolint:govet
+		fs.Log(nil, Color(terminal.YellowFg, "WARNING: Modtime compare was requested but at least one remote does not support it. It is recommended to use --checksum or --size-only instead."))
 	}
 	if (ci.CheckSum || b.opt.Compare.Checksum) && b.opt.IgnoreListingChecksum {
 		if (b.opt.Compare.HashType1 == hash.None || b.opt.Compare.HashType2 == hash.None) && !b.opt.Compare.DownloadHash {
 			fs.Logf(nil, Color(terminal.YellowFg, `WARNING: Checksum compare was requested but at least one remote does not support checksums (or checksums are being ignored) and --ignore-listing-checksum is set.
 			 Ignoring Checksums globally and falling back to --compare modtime,size for sync. (Use --compare size or --size-only to ignore modtime). Path1 (%s): %s, Path2 (%s): %s`),
-				b.fs1.String(), b.opt.Compare.HashType1.String(), b.fs2.String(), b.opt.Compare.HashType2.String()) //nolint:govet
+				b.fs1.String(), b.opt.Compare.HashType1.String(), b.fs2.String(), b.opt.Compare.HashType2.String())
 			b.opt.Compare.Modtime = true
 			b.opt.Compare.Size = true
 			ci.CheckSum = false
 			b.opt.Compare.Checksum = false
 		} else {
-			fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksum for deltas as --ignore-listing-checksum is set")) //nolint:govet
+			fs.Log(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksum for deltas as --ignore-listing-checksum is set"))
 			// note: --checksum will still affect the internal sync calls
 		}
 	}
 	if !ci.CheckSum && !b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum {
-		fs.Infof(nil, Color(terminal.Dim, "Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.")) //nolint:govet
+		fs.Infoc(nil, Color(terminal.Dim, "Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set."))
 		b.opt.IgnoreListingChecksum = true
 	}
 	if !b.opt.Compare.Size && !b.opt.Compare.Modtime && !b.opt.Compare.Checksum {
-		return errors.New(Color(terminal.RedFg, "must set a Compare method. (size, modtime, and checksum can't all be false.)")) //nolint:govet
+		return errors.New(Color(terminal.RedFg, "must set a Compare method. (size, modtime, and checksum can't all be false.)"))
 	}
 
 	notSupported := func(label string, value bool, opt *bool) {
 		if value {
-			fs.Logf(nil, Color(terminal.YellowFg, "WARNING: %s is set but bisync does not support it. It will be ignored."), label) //nolint:govet
+			fs.Logf(nil, Color(terminal.YellowFg, "WARNING: %s is set but bisync does not support it. It will be ignored."), label)
 			*opt = false
 		}
 	}
@@ -123,13 +123,13 @@ func sizeDiffers(a, b int64) bool {
 func hashDiffers(a, b string, ht1, ht2 hash.Type, size1, size2 int64) bool {
 	if a == "" || b == "" {
 		if ht1 != hash.None && ht2 != hash.None && !(size1 <= 0 || size2 <= 0) {
-			fs.Logf(nil, Color(terminal.YellowFg, "WARNING: hash unexpectedly blank despite Fs support (%s, %s) (you may need to --resync!)"), a, b) //nolint:govet
+			fs.Logf(nil, Color(terminal.YellowFg, "WARNING: hash unexpectedly blank despite Fs support (%s, %s) (you may need to --resync!)"), a, b)
 		}
 		return false
 	}
 	if ht1 != ht2 {
 		if !(downloadHash && ((ht1 == hash.MD5 && ht2 == hash.None) || (ht1 == hash.None && ht2 == hash.MD5))) {
-			fs.Infof(nil, Color(terminal.YellowFg, "WARNING: Can't compare hashes of different types (%s, %s)"), ht1.String(), ht2.String()) //nolint:govet
+			fs.Infof(nil, Color(terminal.YellowFg, "WARNING: Can't compare hashes of different types (%s, %s)"), ht1.String(), ht2.String())
 			return false
 		}
 	}
@@ -151,7 +151,7 @@ func (b *bisyncRun) setHashType(ci *fs.ConfigInfo) {
 				return
 			}
 		} else if b.opt.Compare.SlowHashSyncOnly && b.opt.Compare.SlowHashDetected {
-			fs.Logf(b.fs2, Color(terminal.YellowFg, "Ignoring --slow-hash-sync-only and falling back to --no-slow-hash as Path1 and Path2 have no hashes in common.")) //nolint:govet
+			fs.Log(b.fs2, Color(terminal.YellowFg, "Ignoring --slow-hash-sync-only and falling back to --no-slow-hash as Path1 and Path2 have no hashes in common."))
 			b.opt.Compare.SlowHashSyncOnly = false
 			b.opt.Compare.NoSlowHash = true
 			ci.CheckSum = false
@@ -159,7 +159,7 @@ func (b *bisyncRun) setHashType(ci *fs.ConfigInfo) {
 	}
 
 	if !b.opt.Compare.DownloadHash && !b.opt.Compare.SlowHashSyncOnly {
-		fs.Logf(b.fs2, Color(terminal.YellowFg, "--checksum is in use but Path1 and Path2 have no hashes in common; falling back to --compare modtime,size for sync. (Use --compare size or --size-only to ignore modtime)")) //nolint:govet
+		fs.Log(b.fs2, Color(terminal.YellowFg, "--checksum is in use but Path1 and Path2 have no hashes in common; falling back to --compare modtime,size for sync. (Use --compare size or --size-only to ignore modtime)"))
 		fs.Infof("Path1 hashes", "%v", b.fs1.Hashes().String())
 		fs.Infof("Path2 hashes", "%v", b.fs2.Hashes().String())
 		b.opt.Compare.Modtime = true
@@ -167,25 +167,25 @@ func (b *bisyncRun) setHashType(ci *fs.ConfigInfo) {
 		ci.CheckSum = false
 	}
 	if (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.fs1.Features().SlowHash {
-		fs.Infof(nil, Color(terminal.YellowFg, "Slow hash detected on Path1. Will ignore checksum due to slow-hash settings")) //nolint:govet
+		fs.Infoc(nil, Color(terminal.YellowFg, "Slow hash detected on Path1. Will ignore checksum due to slow-hash settings"))
 		b.opt.Compare.HashType1 = hash.None
 	} else {
 		b.opt.Compare.HashType1 = b.fs1.Hashes().GetOne()
 		if b.opt.Compare.HashType1 != hash.None {
-			fs.Logf(b.fs1, Color(terminal.YellowFg, "will use %s for same-side diffs on Path1 only"), b.opt.Compare.HashType1) //nolint:govet
+			fs.Logf(b.fs1, Color(terminal.YellowFg, "will use %s for same-side diffs on Path1 only"), b.opt.Compare.HashType1)
 		}
 	}
 	if (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.fs2.Features().SlowHash {
-		fs.Infof(nil, Color(terminal.YellowFg, "Slow hash detected on Path2. Will ignore checksum due to slow-hash settings")) //nolint:govet
+		fs.Infoc(nil, Color(terminal.YellowFg, "Slow hash detected on Path2. Will ignore checksum due to slow-hash settings"))
 		b.opt.Compare.HashType1 = hash.None
 	} else {
 		b.opt.Compare.HashType2 = b.fs2.Hashes().GetOne()
 		if b.opt.Compare.HashType2 != hash.None {
-			fs.Logf(b.fs2, Color(terminal.YellowFg, "will use %s for same-side diffs on Path2 only"), b.opt.Compare.HashType2) //nolint:govet
+			fs.Logf(b.fs2, Color(terminal.YellowFg, "will use %s for same-side diffs on Path2 only"), b.opt.Compare.HashType2)
 		}
 	}
 	if b.opt.Compare.HashType1 == hash.None && b.opt.Compare.HashType2 == hash.None && !b.opt.Compare.DownloadHash {
-		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksums globally as hashes are ignored or unavailable on both sides.")) //nolint:govet
+		fs.Log(nil, Color(terminal.YellowFg, "WARNING: Ignoring checksums globally as hashes are ignored or unavailable on both sides."))
 		b.opt.Compare.Checksum = false
 		ci.CheckSum = false
 		b.opt.IgnoreListingChecksum = true
@@ -218,7 +218,7 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {
 	if b.opt.CompareFlag == "" {
 		return nil
 	}
-	var CompareFlag CompareOpt // for exlcusions
+	var CompareFlag CompareOpt // for exclusions
 	opts := strings.Split(b.opt.CompareFlag, ",")
 	for _, opt := range opts {
 		switch strings.ToLower(strings.TrimSpace(opt)) {
@@ -232,7 +232,7 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {
 			b.opt.Compare.Checksum = true
 			CompareFlag.Checksum = true
 		default:
-			return fmt.Errorf(Color(terminal.RedFg, "unknown compare option: %s (must be size, modtime, or checksum)"), opt) //nolint:govet
+			return fmt.Errorf(Color(terminal.RedFg, "unknown compare option: %s (must be size, modtime, or checksum)"), opt)
 		}
 	}
 
@@ -284,14 +284,14 @@ func tryDownloadHash(ctx context.Context, o fs.DirEntry, hashVal string) (string
 	}
 	if o.Size() < 0 {
 		downloadHashWarn.Do(func() {
-			fs.Logf(o, Color(terminal.YellowFg, "Skipping hash download as checksum not reliable with files of unknown length.")) //nolint:govet
+			fs.Log(o, Color(terminal.YellowFg, "Skipping hash download as checksum not reliable with files of unknown length."))
 		})
 		fs.Debugf(o, "Skipping hash download as checksum not reliable with files of unknown length.")
 		return hashVal, hash.ErrUnsupported
 	}
 
 	firstDownloadHash.Do(func() {
-		fs.Infof(obj.Fs().Name(), Color(terminal.Dim, "Downloading hashes...")) //nolint:govet
+		fs.Infoc(obj.Fs().Name(), Color(terminal.Dim, "Downloading hashes..."))
 	})
 	tr := accounting.Stats(ctx).NewCheckingTransfer(o, "computing hash with --download-hash")
 	defer func() {

cmd/bisync/deltas.go

@@ -161,9 +161,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 		return
 	}
 
-	if err == nil {
-		err = b.checkListing(now, newListing, "current "+msg)
-	}
+	err = b.checkListing(now, newListing, "current "+msg)
 	if err != nil {
 		return
 	}
@@ -286,7 +284,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 }
 
 // applyDeltas
-func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changes1, changes2 bool, results2to1, results1to2 []Results, queues queues, err error) {
+func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (results2to1, results1to2 []Results, queues queues, err error) {
 	path1 := bilib.FsPath(b.fs1)
 	path2 := bilib.FsPath(b.fs2)
 
@@ -367,7 +365,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		}
 	}
 
-	//if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
+	// if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
 	matches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
 
 	for _, file := range ds1.sort() {
@@ -392,7 +390,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 			} else if d2.is(deltaOther) {
 				b.indent("!WARNING", file, "New or changed in both paths")
 
-				//if files are identical, leave them alone instead of renaming
+				// if files are identical, leave them alone instead of renaming
 				if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {
 					fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
 					ls1.getPut(file, skippedDirs1)
@@ -486,7 +484,6 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 
 	// Do the batch operation
 	if copy2to1.NotEmpty() && !b.InGracefulShutdown {
-		changes1 = true
 		b.indent("Path2", "Path1", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 1)
 		results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")
@@ -498,12 +495,11 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 			return
 		}
 
-		//copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		// copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, &results2to1, "make")
 	}
 
 	if copy1to2.NotEmpty() && !b.InGracefulShutdown {
-		changes2 = true
 		b.indent("Path1", "Path2", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 2)
 		results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")
@@ -515,7 +511,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 			return
 		}
 
-		//copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		// copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, &results1to2, "make")
 	}
 
@@ -523,7 +519,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err = b.saveQueue(delete1, "delete1"); err != nil {
 			return
 		}
-		//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		// propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, &results2to1, "remove")
 	}
 
@@ -531,7 +527,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 		if err = b.saveQueue(delete2, "delete2"); err != nil {
 			return
 		}
-		//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		// propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, &results1to2, "remove")
 	}
 

cmd/bisync/listing.go

@@ -394,7 +394,7 @@ func parseHash(str string) (string, string, error) {
 	return "", "", fmt.Errorf("invalid hash %q", str)
 }
 
-// checkListing verifies that listing is not empty (unless resynching)
+// checkListing verifies that listing is not empty (unless resyncing)
 func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error {
 	if b.opt.Resync || !ls.empty() {
 		return nil

cmd/bisync/log.go

@@ -78,6 +78,15 @@ func Color(style string, s string) string {
 	return style + s + terminal.Reset
 }
 
+// ColorX handles terminal colors for bisync
+func ColorX(style string, s string) string {
+	if !Colors {
+		return s
+	}
+	terminal.Start()
+	return style + s + terminal.Reset
+}
+
 func encode(s string) string {
 	return encoder.OS.ToStandardPath(encoder.OS.FromStandardPath(s))
 }

cmd/bisync/operations.go

@@ -131,18 +131,18 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 		finaliseOnce.Do(func() {
 			if atexit.Signalled() {
 				if b.opt.Resync {
-					fs.Logf(nil, Color(terminal.GreenFg, "No need to gracefully shutdown during --resync (just run it again.)")) //nolint:govet
+					fs.Log(nil, Color(terminal.GreenFg, "No need to gracefully shutdown during --resync (just run it again.)"))
 				} else {
-					fs.Logf(nil, Color(terminal.YellowFg, "Attempting to gracefully shutdown. (Send exit signal again for immediate un-graceful shutdown.)")) //nolint:govet
+					fs.Log(nil, Color(terminal.YellowFg, "Attempting to gracefully shutdown. (Send exit signal again for immediate un-graceful shutdown.)"))
 					b.InGracefulShutdown = true
 					if b.SyncCI != nil {
-						fs.Infof(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early.")) //nolint:govet
+						fs.Infoc(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early."))
 						b.SyncCI.MaxTransfer = 1
 						b.SyncCI.MaxDuration = 1 * time.Second
 						b.SyncCI.CutoffMode = fs.CutoffModeSoft
 						gracePeriod := 30 * time.Second // TODO: flag to customize this?
 						if !waitFor("Canceling Sync if not done in", gracePeriod, func() bool { return b.CleanupCompleted }) {
-							fs.Logf(nil, Color(terminal.YellowFg, "Canceling sync and cleaning up")) //nolint:govet
+							fs.Log(nil, Color(terminal.YellowFg, "Canceling sync and cleaning up"))
 							b.CancelSync()
 							waitFor("Aborting Bisync if not done in", 60*time.Second, func() bool { return b.CleanupCompleted })
 						}
@@ -150,13 +150,13 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 						// we haven't started to sync yet, so we're good.
 						// no need to worry about the listing files, as we haven't overwritten them yet.
 						b.CleanupCompleted = true
-						fs.Logf(nil, Color(terminal.GreenFg, "Graceful shutdown completed successfully.")) //nolint:govet
+						fs.Log(nil, Color(terminal.GreenFg, "Graceful shutdown completed successfully."))
 					}
 				}
 				if !b.CleanupCompleted {
 					if !b.opt.Resync {
-						fs.Logf(nil, Color(terminal.HiRedFg, "Graceful shutdown failed."))                       //nolint:govet
-						fs.Logf(nil, Color(terminal.RedFg, "Bisync interrupted. Must run --resync to recover.")) //nolint:govet
+						fs.Log(nil, Color(terminal.HiRedFg, "Graceful shutdown failed."))
+						fs.Log(nil, Color(terminal.RedFg, "Bisync interrupted. Must run --resync to recover."))
 					}
 					markFailed(b.listing1)
 					markFailed(b.listing2)
@@ -180,14 +180,14 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 			b.critical = false
 		}
 		if err == nil {
-			fs.Logf(nil, Color(terminal.GreenFg, "Graceful shutdown completed successfully.")) //nolint:govet
+			fs.Log(nil, Color(terminal.GreenFg, "Graceful shutdown completed successfully."))
 		}
 	}
 
 	if b.critical {
 		if b.retryable && b.opt.Resilient {
-			fs.Errorf(nil, Color(terminal.RedFg, "Bisync critical error: %v"), err)                                                  //nolint:govet
-			fs.Errorf(nil, Color(terminal.YellowFg, "Bisync aborted. Error is retryable without --resync due to --resilient mode.")) //nolint:govet
+			fs.Errorf(nil, Color(terminal.RedFg, "Bisync critical error: %v"), err)
+			fs.Error(nil, Color(terminal.YellowFg, "Bisync aborted. Error is retryable without --resync due to --resilient mode."))
 		} else {
 			if bilib.FileExists(b.listing1) {
 				_ = os.Rename(b.listing1, b.listing1+"-err")
@@ -196,15 +196,15 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 				_ = os.Rename(b.listing2, b.listing2+"-err")
 			}
 			fs.Errorf(nil, Color(terminal.RedFg, "Bisync critical error: %v"), err)
-			fs.Errorf(nil, Color(terminal.RedFg, "Bisync aborted. Must run --resync to recover.")) //nolint:govet
+			fs.Error(nil, Color(terminal.RedFg, "Bisync aborted. Must run --resync to recover."))
 		}
 		return ErrBisyncAborted
 	}
 	if b.abort && !b.InGracefulShutdown {
-		fs.Logf(nil, Color(terminal.RedFg, "Bisync aborted. Please try again.")) //nolint:govet
+		fs.Log(nil, Color(terminal.RedFg, "Bisync aborted. Please try again."))
 	}
 	if err == nil {
-		fs.Infof(nil, Color(terminal.GreenFg, "Bisync successful")) //nolint:govet
+		fs.Infoc(nil, Color(terminal.GreenFg, "Bisync successful"))
 	}
 	return err
 }
@@ -270,7 +270,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 		if b.opt.Recover && bilib.FileExists(b.listing1+"-old") && bilib.FileExists(b.listing2+"-old") {
 			errTip := fmt.Sprintf(Color(terminal.CyanFg, "Path1: %s\n"), Color(terminal.HiBlueFg, b.listing1))
 			errTip += fmt.Sprintf(Color(terminal.CyanFg, "Path2: %s"), Color(terminal.HiBlueFg, b.listing2))
-			fs.Logf(nil, Color(terminal.YellowFg, "Listings not found. Reverting to prior backup as --recover is set. \n")+errTip) //nolint:govet
+			fs.Log(nil, Color(terminal.YellowFg, "Listings not found. Reverting to prior backup as --recover is set. \n")+errTip)
 			if opt.CheckSync != CheckSyncFalse {
 				// Run CheckSync to ensure old listing is valid (garbage in, garbage out!)
 				fs.Infof(nil, "Validating backup listings for Path1 %s vs Path2 %s", quotePath(path1), quotePath(path2))
@@ -279,7 +279,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 					b.retryable = true
 					return err
 				}
-				fs.Infof(nil, Color(terminal.GreenFg, "Backup listing is valid.")) //nolint:govet
+				fs.Infoc(nil, Color(terminal.GreenFg, "Backup listing is valid."))
 			}
 			b.revertToOldListings()
 		} else {
@@ -299,7 +299,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 	fs.Infof(nil, "Building Path1 and Path2 listings")
 	ls1, ls2, err = b.makeMarchListing(fctx)
 	if err != nil || accounting.Stats(fctx).Errored() {
-		fs.Errorf(nil, Color(terminal.RedFg, "There were errors while building listings. Aborting as it is too dangerous to continue.")) //nolint:govet
+		fs.Error(nil, Color(terminal.RedFg, "There were errors while building listings. Aborting as it is too dangerous to continue."))
 		b.critical = true
 		b.retryable = true
 		return err
@@ -359,8 +359,6 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 
 	// Determine and apply changes to Path1 and Path2
 	noChanges := ds1.empty() && ds2.empty()
-	changes1 := false // 2to1
-	changes2 := false // 1to2
 	results2to1 := []Results{}
 	results1to2 := []Results{}
 
@@ -370,7 +368,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 		fs.Infof(nil, "No changes found")
 	} else {
 		fs.Infof(nil, "Applying changes")
-		changes1, changes2, results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
+		results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
 		if err != nil {
 			if b.InGracefulShutdown && (err == context.Canceled || err == accounting.ErrorMaxTransferLimitReachedGraceful || strings.Contains(err.Error(), "context canceled")) {
 				fs.Infof(nil, "Ignoring sync error due to Graceful Shutdown: %v", err)
@@ -395,21 +393,11 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 	}
 	b.saveOldListings()
 	// save new listings
-	// NOTE: "changes" in this case does not mean this run vs. last run, it means start of this run vs. end of this run.
-	// i.e. whether we can use the March lst-new as this side's lst without modifying it.
 	if noChanges {
 		b.replaceCurrentListings()
 	} else {
-		if changes1 || b.InGracefulShutdown { // 2to1
-			err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false)
-		} else {
-			err1 = bilib.CopyFileIfExists(b.newListing1, b.listing1)
-		}
-		if changes2 || b.InGracefulShutdown { // 1to2
-			err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)
-		} else {
-			err2 = bilib.CopyFileIfExists(b.newListing2, b.listing2)
-		}
+		err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) // 2to1
+		err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)  // 1to2
 	}
 	if b.DebugName != "" {
 		l1, _ := b.loadListing(b.listing1)
@@ -623,7 +611,7 @@ func (b *bisyncRun) checkSyntax() error {
 
 func (b *bisyncRun) debug(nametocheck, msgiftrue string) {
 	if b.DebugName != "" && b.DebugName == nametocheck {
-		fs.Infof(Color(terminal.MagentaBg, "DEBUGNAME "+b.DebugName), Color(terminal.MagentaBg, msgiftrue)) //nolint:govet
+		fs.Infoc(Color(terminal.MagentaBg, "DEBUGNAME "+b.DebugName), Color(terminal.MagentaBg, msgiftrue))
 	}
 }
 

cmd/bisync/queue.go

@@ -161,7 +161,7 @@ func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEn
 		prettyprint(result, "writing result", fs.LogLevelDebug)
 		if result.Size < 0 && result.Flags != "d" && ((queueCI.CheckSum && !downloadHash) || queueCI.SizeOnly) {
 			once.Do(func() {
-				fs.Logf(result.Name, Color(terminal.YellowFg, "Files of unknown size (such as Google Docs) do not sync reliably with --checksum or --size-only. Consider using modtime instead (the default) or --drive-skip-gdocs")) //nolint:govet
+				fs.Log(result.Name, Color(terminal.YellowFg, "Files of unknown size (such as Google Docs) do not sync reliably with --checksum or --size-only. Consider using modtime instead (the default) or --drive-skip-gdocs"))
 			})
 		}
 

cmd/bisync/resolve.go

@@ -142,7 +142,7 @@ func (b *bisyncRun) resolve(ctxMove context.Context, path1, path2, file, alias s
 		if winningPath > 0 {
 			fs.Infof(file, Color(terminal.GreenFg, "The winner is: Path%d"), winningPath)
 		} else {
-			fs.Infof(file, Color(terminal.RedFg, "A winner could not be determined.")) //nolint:govet
+			fs.Infoc(file, Color(terminal.RedFg, "A winner could not be determined."))
 		}
 	}
 

cmd/bisync/resync.go

@@ -15,7 +15,7 @@ import (
 // and either flag is sufficient without the other.
 func (b *bisyncRun) setResyncDefaults() {
 	if b.opt.Resync && b.opt.ResyncMode == PreferNone {
-		fs.Debugf(nil, Color(terminal.Dim, "defaulting to --resync-mode path1 as --resync is set")) //nolint:govet
+		fs.Debug(nil, Color(terminal.Dim, "defaulting to --resync-mode path1 as --resync is set"))
 		b.opt.ResyncMode = PreferPath1
 	}
 	if b.opt.ResyncMode != PreferNone {

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -0,0 +1 @@
+"file1.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old

@@ -0,0 +1,10 @@
+# bisync listing v1 from test
+-      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+-       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
+-        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/test.log

@@ -0,0 +1,73 @@
+(01)  : test concurrent
+
+(02)  : test initial bisync
+(03)  : bisync resync
+INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
+INFO  : Bisyncing with Comparison Settings:
+{
+"Modtime": true,
+"Size": true,
+"Checksum": false,
+"NoSlowHash": false,
+"SlowHashSyncOnly": false,
+"DownloadHash": false
+}
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Copying Path2 files to Path1
+INFO  : - Path2    Resync is copying files to         - Path1
+INFO  : - Path1    Resync is copying files to         - Path2
+INFO  : Resync updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(04)  : test changed on one path - file1
+(05)  : touch-glob 2001-01-02 {datadir/} file5R.txt
+(06)  : touch-glob 2023-08-26 {datadir/} file7.txt
+(07)  : copy-as {datadir/}file5R.txt {path2/} file1.txt
+
+(08)  : test bisync with file changed during
+(09)  : concurrent-func
+(10)  : bisync
+INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
+INFO  : Bisyncing with Comparison Settings:
+{
+"Modtime": true,
+"Size": true,
+"Checksum": false,
+"NoSlowHash": false,
+"SlowHashSyncOnly": false,
+"DownloadHash": false
+}
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Building Path1 and Path2 listings
+INFO  : Path1 checking for diffs
+INFO  : Path2 checking for diffs
+INFO  : - Path2    File changed: size (larger), time (newer) - file1.txt
+INFO  : Path2:    1 changes:    0 new,    1 modified,    0 deleted
+INFO  : (Modified:    1 newer,    0 older,    1 larger,    0 smaller)
+INFO  : Applying changes
+INFO  : - Path2    Queue copy to Path1       - {path1/}file1.txt
+INFO  : - Path2    Do queued copies to                - Path1
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful
+
+(11)  : bisync
+INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
+INFO  : Bisyncing with Comparison Settings:
+{
+"Modtime": true,
+"Size": true,
+"Checksum": false,
+"NoSlowHash": false,
+"SlowHashSyncOnly": false,
+"DownloadHash": false
+}
+INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO  : Building Path1 and Path2 listings
+INFO  : Path1 checking for diffs
+INFO  : Path2 checking for diffs
+INFO  : No changes found
+INFO  : Updating listings
+INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO  : Bisync successful

cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST

@@ -0,0 +1 @@
+This file is used for testing the health of rclone accesses to the local/remote file system.  Do not delete.

cmd/bisync/testdata/test_concurrent/modfiles/file1.txt

@@ -0,0 +1 @@
+This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file10.txt

@@ -0,0 +1 @@
+This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file11.txt

@@ -0,0 +1 @@
+This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file2.txt

@@ -0,0 +1 @@
+Newer version

cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt

@@ -0,0 +1 @@
+This file is newer and not equal to 5R

cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt

@@ -0,0 +1 @@
+This file is newer and not equal to 5L

cmd/bisync/testdata/test_concurrent/modfiles/file6.txt

@@ -0,0 +1 @@
+This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file7.txt

@@ -0,0 +1 @@
+This file is newer

cmd/bisync/testdata/test_concurrent/scenario.txt

@@ -0,0 +1,15 @@
+test concurrent
+
+test initial bisync
+bisync resync
+
+test changed on one path - file1
+touch-glob 2001-01-02 {datadir/} file5R.txt
+touch-glob 2023-08-26 {datadir/} file7.txt
+copy-as {datadir/}file5R.txt {path2/} file1.txt
+
+test bisync with file changed during
+concurrent-func
+bisync
+
+bisync

cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log

@@ -80,6 +80,7 @@ INFO  : Path2 checking for diffs
 INFO  : Applying changes
 INFO  : - Path1    Queue copy to Path2       - {path2/}subdir
 INFO  : - Path1    Do queued copies to                - Path2
+INFO  : subdir: Making directory
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_resync/golden/test.log

@@ -23,7 +23,7 @@ INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful
 (05)  : move-listings empty-path1
 
-(06)  : test 2. resync with empty path2, resulting in synching all content to path2.
+(06)  : test 2. resync with empty path2, resulting in syncing all content to path2.
 (07)  : purge-children {path2/}
 (08)  : bisync resync
 INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.

cmd/bisync/testdata/test_resync/scenario.txt

@@ -1,6 +1,6 @@
 test resync
 # 1. Resync with empty Path1, resulting in copying all content FROM Path2
-# 2. Resync with empty Path2, resulting in synching all content TO Path2
+# 2. Resync with empty Path2, resulting in syncing all content TO Path2
 # 3. Exercise all of the various file difference scenarios during a resync:
 #    File       Path1       Path2           Expected action     Who wins
 #  - file1.txt  Exists      Missing         Sync Path1 >Path2   Path1
@@ -17,7 +17,7 @@ purge-children {path1/}
 bisync resync
 move-listings empty-path1
 
-test 2. resync with empty path2, resulting in synching all content to path2.
+test 2. resync with empty path2, resulting in syncing all content to path2.
 purge-children {path2/}
 bisync resync
 move-listings empty-path2

cmd/cmd.go

@@ -429,11 +429,12 @@ func initConfig() {
 		fs.Fatalf(nil, "Failed to start remote control: %v", err)
 	}
 
-	// Start the metrics server if configured
-	_, err = rcserver.MetricsStart(ctx, &rc.Opt)
-	if err != nil {
-		fs.Fatalf(nil, "Failed to start metrics server: %v", err)
-
+	// Start the metrics server if configured and not running the "rc" command
+	if len(os.Args) >= 2 && os.Args[1] != "rc" {
+		_, err = rcserver.MetricsStart(ctx, &rc.Opt)
+		if err != nil {
+			fs.Fatalf(nil, "Failed to start metrics server: %v", err)
+		}
 	}
 
 	// Setup CPU profiling if desired

cmd/cmount/fs.go

@@ -121,19 +121,6 @@ func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *vfs.Dir, err
 	return leaf, dir, errc
 }
 
-// lookup a File given a path
-func (fsys *FS) lookupFile(path string) (file *vfs.File, errc int) {
-	node, errc := fsys.lookupNode(path)
-	if errc != 0 {
-		return nil, errc
-	}
-	file, ok := node.(*vfs.File)
-	if !ok {
-		return nil, -fuse.EISDIR
-	}
-	return file, 0
-}
-
 // get a node and handle from the path or from the fh if not fhUnset
 //
 // handle may be nil
@@ -154,15 +141,9 @@ func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
 	Size := uint64(node.Size())
 	Blocks := (Size + 511) / 512
 	modTime := node.ModTime()
-	Mode := node.Mode().Perm()
-	if node.IsDir() {
-		Mode |= fuse.S_IFDIR
-	} else {
-		Mode |= fuse.S_IFREG
-	}
 	//stat.Dev = 1
 	stat.Ino = node.Inode() // FIXME do we need to set the inode number?
-	stat.Mode = uint32(Mode)
+	stat.Mode = getMode(node)
 	stat.Nlink = 1
 	stat.Uid = fsys.VFS.Opt.UID
 	stat.Gid = fsys.VFS.Opt.GID
@@ -509,14 +490,15 @@ func (fsys *FS) Link(oldpath string, newpath string) (errc int) {
 
 // Symlink creates a symbolic link.
 func (fsys *FS) Symlink(target string, newpath string) (errc int) {
-	defer log.Trace(target, "newpath=%q", newpath)("errc=%d", &errc)
-	return -fuse.ENOSYS
+	defer log.Trace(target, "newpath=%q, target=%q", newpath, target)("errc=%d", &errc)
+	return translateError(fsys.VFS.Symlink(target, newpath))
 }
 
 // Readlink reads the target of a symbolic link.
 func (fsys *FS) Readlink(path string) (errc int, linkPath string) {
-	defer log.Trace(path, "")("linkPath=%q, errc=%d", &linkPath, &errc)
-	return -fuse.ENOSYS, ""
+	defer log.Trace(path, "")("errc=%v, linkPath=%q", &errc, linkPath)
+	linkPath, err := fsys.VFS.Readlink(path)
+	return translateError(err), linkPath
 }
 
 // Chmod changes the permission bits of a file.
@@ -580,7 +562,7 @@ func (fsys *FS) Getpath(path string, fh uint64) (errc int, normalisedPath string
 		return errc, ""
 	}
 	normalisedPath = node.Path()
-	if !strings.HasPrefix("/", normalisedPath) {
+	if !strings.HasPrefix(normalisedPath, "/") {
 		normalisedPath = "/" + normalisedPath
 	}
 	return 0, normalisedPath
@@ -615,6 +597,8 @@ func translateError(err error) (errc int) {
 		return -fuse.ENOSYS
 	case vfs.EINVAL:
 		return -fuse.EINVAL
+	case vfs.ELOOP:
+		return -fuse.ELOOP
 	}
 	fs.Errorf(nil, "IO error: %v", err)
 	return -fuse.EIO
@@ -646,6 +630,22 @@ func translateOpenFlags(inFlags int) (outFlags int) {
 	return outFlags
 }
 
+// get the Mode from a vfs Node
+func getMode(node os.FileInfo) uint32 {
+	vfsMode := node.Mode()
+	Mode := vfsMode.Perm()
+	if vfsMode&os.ModeDir != 0 {
+		Mode |= fuse.S_IFDIR
+	} else if vfsMode&os.ModeSymlink != 0 {
+		Mode |= fuse.S_IFLNK
+	} else if vfsMode&os.ModeNamedPipe != 0 {
+		Mode |= fuse.S_IFIFO
+	} else {
+		Mode |= fuse.S_IFREG
+	}
+	return uint32(Mode)
+}
+
 // Make sure interfaces are satisfied
 var (
 	_ fuse.FileSystemInterface = (*FS)(nil)

cmd/cmount/mount.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"os"
 	"runtime"
-	"strings"
 	"time"
 
 	"github.com/rclone/rclone/cmd/mountlib"
@@ -35,19 +34,6 @@ func init() {
 	buildinfo.Tags = append(buildinfo.Tags, "cmount")
 }
 
-// Find the option string in the current options
-func findOption(name string, options []string) (found bool) {
-	for _, option := range options {
-		if option == "-o" {
-			continue
-		}
-		if strings.Contains(option, name) {
-			return true
-		}
-	}
-	return false
-}
-
 // mountOptions configures the options from the command line flags
 func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
 	// Options
@@ -93,9 +79,9 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.
 		if VFS.Opt.ReadOnly {
 			options = append(options, "-o", "ro")
 		}
-		if opt.WritebackCache {
-			// FIXME? options = append(options, "-o", WritebackCache())
-		}
+		//if opt.WritebackCache {
+		// FIXME? options = append(options, "-o", WritebackCache())
+		//}
 		if runtime.GOOS == "darwin" {
 			if opt.VolumeName != "" {
 				options = append(options, "-o", "volname="+opt.VolumeName)
@@ -111,9 +97,7 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.
 	for _, option := range opt.ExtraOptions {
 		options = append(options, "-o", option)
 	}
-	for _, option := range opt.ExtraFlags {
-		options = append(options, option)
-	}
+	options = append(options, opt.ExtraFlags...)
 	return options
 }
 

cmd/config/config.go

@@ -303,6 +303,9 @@ func doConfig(name string, in rc.Params, do func(config.UpdateRemoteOpt) (*fs.Co
 	if err != nil {
 		return err
 	}
+	if updateRemoteOpt.NoOutput {
+		return nil
+	}
 	if !(updateRemoteOpt.NonInteractive || updateRemoteOpt.Continue) {
 		config.ShowRemote(name)
 	} else {
@@ -323,6 +326,7 @@ func init() {
 	for _, cmdFlags := range []*pflag.FlagSet{configCreateCommand.Flags(), configUpdateCommand.Flags()} {
 		flags.BoolVarP(cmdFlags, &updateRemoteOpt.Obscure, "obscure", "", false, "Force any passwords to be obscured", "Config")
 		flags.BoolVarP(cmdFlags, &updateRemoteOpt.NoObscure, "no-obscure", "", false, "Force any passwords not to be obscured", "Config")
+		flags.BoolVarP(cmdFlags, &updateRemoteOpt.NoOutput, "no-output", "", false, "Don't provide any output", "Config")
 		flags.BoolVarP(cmdFlags, &updateRemoteOpt.NonInteractive, "non-interactive", "", false, "Don't interact with user and return questions", "Config")
 		flags.BoolVarP(cmdFlags, &updateRemoteOpt.Continue, "continue", "", false, "Continue the configuration process with an answer", "Config")
 		flags.BoolVarP(cmdFlags, &updateRemoteOpt.All, "all", "", false, "Ask the full set of config questions", "Config")
@@ -549,12 +553,12 @@ password to re-encrypt the config.
 
 When |--password-command| is called to change the password then the
 environment variable |RCLONE_PASSWORD_CHANGE=1| will be set. So if
-changing passwords programatically you can use the environment
+changing passwords programmatically you can use the environment
 variable to distinguish which password you must supply.
 
 Alternatively you can remove the password first (with |rclone config
 encryption remove|), then set it again with this command which may be
-easier if you don't mind the unecrypted config file being on the disk
+easier if you don't mind the unencrypted config file being on the disk
 briefly.
 `, "|", "`"),
 	RunE: func(command *cobra.Command, args []string) error {

cmd/copyurl/copyurl.go

@@ -54,7 +54,7 @@ destination if there is one with the same name.
 Setting |--stdout| or making the output file name |-|
 will cause the output to be written to standard output.
 
-### Troublshooting
+### Troubleshooting
 
 If you can't get |rclone copyurl| to work then here are some things you can try: