convmv: update help text - FIXME WIP

need a help line for each conversion mode

.github/workflows/build.yml

@@ -26,12 +26,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.23']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -42,14 +42,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -58,14 +58,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -75,20 +75,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '>=1.24.0-rc.1'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.21
+          - job_name: go1.23
             os: ubuntu-latest
-            go: '1.21'
-            quicktest: true
-            racequicktest: true
-
-          - job_name: go1.22
-            os: ubuntu-latest
-            go: '1.22'
+            go: '1.23'
             quicktest: true
             racequicktest: true
 
@@ -311,7 +305,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '>=1.24.0-rc.1'
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_beta_docker_image.yml

@@ -1,77 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKERHUB_USERNAME }}
-                password: ${{ secrets.DOCKERHUB_TOKEN }}
-            - name: Extract metadata (tags, labels) for Docker
-              id: meta
-              uses: docker/metadata-action@v5
-              with:
-                images: ghcr.io/${{ github.repository }}
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to GitHub Container Registry
-              uses: docker/login-action@v3
-              with:
-                registry: ghcr.io
-                # This is the user that triggered the Workflow. In this case, it will
-                # either be the user whom created the Release or manually triggered
-                # the workflow_dispatch.
-                username: ${{ github.actor }}
-                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
-                # GitHub Actions at the start of a workflow run to identify the job.
-                # This is used to authenticate against GitHub Container Registry.
-                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
-                # for more detailed information.
-                password: ${{ secrets.GITHUB_TOKEN }}
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                push: true # push the image to ghcr
-                tags: |
-                  ghcr.io/rclone/rclone:beta
-                  rclone/rclone:beta
-                labels: ${{ steps.meta.outputs.labels }}
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                cache-from: type=gha, scope=${{ github.workflow }}
-                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
-                provenance: false
-              # Eventually cache will need to be cleared if builds more frequent than once a week
-              # https://github.com/docker/build-push-action/issues/252
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,294 @@
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*-
+
+name: Build & Push Docker Images
+
+# Trigger the workflow on push or pull request
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
+
+jobs:
+  build-image:
+    if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runs-on: ubuntu-24.04
+          - platform: linux/386
+            runs-on: ubuntu-24.04
+          - platform: linux/arm64
+            runs-on: ubuntu-24.04-arm
+          - platform: linux/arm/v7
+            runs-on: ubuntu-24.04-arm
+          - platform: linux/arm/v6
+            runs-on: ubuntu-24.04-arm
+
+    name: Build Docker Image for ${{ matrix.platform }}
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - name: Free Space
+        shell: bash
+        run: |
+          df -h .
+          # Remove android SDK
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove .net runtime
+          sudo rm -rf /usr/share/dotnet || true
+          df -h .
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set REPO_NAME Variable
+        run: |
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
+
+      - name: Set PLATFORM Variable
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Set CACHE_NAME Variable
+        shell: python
+        run: |
+          import os, re
+
+          def slugify(input_string, max_length=63):
+            slug = input_string.lower()
+            slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
+            slug = slug.strip()
+            slug = re.sub(r'\s+', '-', slug)
+            slug = re.sub(r'-+', '-', slug)
+            slug = slug[:max_length]
+            slug = re.sub(r'[-]+$', '', slug)
+            return slug
+
+          ref_name_slug = "cache"
+
+          if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
+            ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"CACHE_NAME={ref_name_slug}\n")
+
+      - name: Get ImageOS
+        # There's no way around this, because "ImageOS" is only available to
+        # processes, but the setup-go action uses it in its key.
+        id: imageos
+        uses: actions/github-script@v7
+        with:
+          result-encoding: string
+          script: |
+            return process.env.ImageOS
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
+        with:
+          images: |
+            ghcr.io/${{ env.REPO_NAME }}
+          labels: |
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=beta,enable={{is_default_branch}}
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load Go Build Cache for Docker
+        id: go-cache
+        uses: actions/cache@v4
+        with:
+          key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
+          # Cache only the go builds, the module download is cached via the docker layer caching
+          path: |
+            go-build-cache
+
+      - name: Inject Go Build Cache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "go-build-cache": "/root/.cache/go-build"
+            }
+          skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Publish Image Digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          file: Dockerfile
+          context: .
+          provenance: false
+          # don't specify 'tags' here (error "get can't push tagged ref by digest")
+          # tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
+          platforms: ${{ matrix.platform }}
+          outputs: |
+            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
+          cache-to: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
+
+      - name: Export Image Digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload Image Digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM }}
+          path: /tmp/digests/*
+          retention-days: 1
+          if-no-files-found: error
+
+  merge-image:
+    name: Merge & Push Final Docker Image
+    runs-on: ubuntu-24.04
+    needs:
+      - build-image
+
+    steps:
+      - name: Download Image Digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set REPO_NAME Variable
+        run: |
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
+        with:
+          images: |
+            ${{ env.REPO_NAME }}
+            ghcr.io/${{ env.REPO_NAME }}
+          labels: |
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=beta,enable={{is_default_branch}}
+
+      - name: Extract Tags
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
+          tags_string = " ".join(tags)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"TAGS={tags_string}\n")
+
+      - name: Extract Annotations
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
+          annotations_string = " ".join(annotations)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"ANNOTATIONS={annotations_string}\n")
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create & Push Manifest List
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create \
+          ${{ env.TAGS }} \
+          ${{ env.ANNOTATIONS }} \
+          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
+
+      - name: Inspect and Run Multi-Platform Image
+        run: |
+          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_docker_plugin.yml

@@ -0,0 +1,49 @@
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*-
+
+name: Release Build for Docker Plugin
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
+
+jobs:
+  build_docker_volume_plugin:
+    if: inputs.manual || github.repository == 'rclone/rclone'
+    name: Build docker plugin job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free some space
+        shell: bash
+        run: |
+          df -h .
+          # Remove android SDK
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove .net runtime
+          sudo rm -rf /usr/share/dotnet || true
+          df -h .
+      - name: Checkout master
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build and publish docker plugin
+        shell: bash
+        run: |
+          VER=${GITHUB_REF#refs/tags/}
+          PLUGIN_USER=rclone
+          docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                       --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+          for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+              export PLUGIN_USER PLUGIN_ARCH
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+          done
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/build_publish_release_docker_image.yml

@@ -1,89 +0,0 @@
-name: Docker release build
-
-on:
-  release:
-    types: [published]
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Get actual patch version
-              id: actual_patch_version
-              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
-            - name: Get actual minor version
-              id: actual_minor_version
-              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
-            - name: Get actual major version
-              id: actual_major_version
-              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKER_HUB_USER }}
-                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                push: true
-                tags: |
-                  rclone/rclone:latest
-                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
-                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
-                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

Dockerfile

@@ -1,19 +1,47 @@
 FROM golang:alpine AS builder
 
-COPY . /go/src/github.com/rclone/rclone/
+ARG CGO_ENABLED=0
+
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN apk add --no-cache make bash gawk git
-RUN \
-  CGO_ENABLED=0 \
-  make
-RUN ./rclone version
+RUN echo "**** Set Go Environment Variables ****" && \
+    go env -w GOCACHE=/root/.cache/go-build
+
+RUN echo "**** Install Dependencies ****" && \
+    apk add --no-cache \
+        make \
+        bash \
+        gawk \
+        git
+
+COPY go.mod .
+COPY go.sum .
+
+RUN echo "**** Download Go Dependencies ****" && \
+    go mod download -x
+
+RUN echo "**** Verify Go Dependencies ****" && \
+    go mod verify
+
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
+    echo "**** Build Binary ****" && \
+    make
+
+RUN echo "**** Print Version Binary ****" && \
+    ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse3 tzdata && \
-  echo "user_allow_other" >> /etc/fuse.conf
+RUN echo "**** Install Dependencies ****" && \
+    apk add --no-cache \
+        ca-certificates \
+        fuse3 \
+        tzdata && \
+    echo "Enable user_allow_other in fuse" && \
+    echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

RELEASE.md

@@ -47,13 +47,20 @@ Early in the next release cycle update the dependencies.
   * `git commit -a -v -m "build: update all dependencies"`
 
 If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
+
+    go 1.22.0
+    
+then go to manual mode. `go1.22` here is the lowest supported version
 in the `go.mod`.
 
+If `make updatedirect` added a `toolchain` directive then remove it.
+We don't want to force a toolchain on our users. Linux packagers are
+often using a version of Go that is a few versions out of date.
+
 ```
 go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
 go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
+go mod tidy -go=1.22 -compat=1.22
 ```
 
 If the `go mod tidy` fails use the output from it to remove the
@@ -124,8 +131,8 @@ Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * Do the steps as above
   * make startstable
+  * Do the steps as above
   * git co master
   * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

VERSION

@@ -1 +1 @@
-v1.69.0
+v1.70.0

backend/azureblob/azureblob_internal_test.go

@@ -3,16 +3,149 @@
 package azureblob
 
 import (
+	"context"
+	"encoding/base64"
+	"strings"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func (f *Fs) InternalTest(t *testing.T) {
-	// Check first feature flags are set on this
-	// remote
+func TestBlockIDCreator(t *testing.T) {
+	// Check creation and random number
+	bic, err := newBlockIDCreator()
+	require.NoError(t, err)
+	bic2, err := newBlockIDCreator()
+	require.NoError(t, err)
+	assert.NotEqual(t, bic.random, bic2.random)
+	assert.NotEqual(t, bic.random, [8]byte{})
+
+	// Set random to known value for tests
+	bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
+	chunkNumber := uint64(0xFEDCBA9876543210)
+
+	// Check creation of ID
+	want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
+	got := bic.newBlockID(chunkNumber)
+	assert.Equal(t, want, got)
+	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)
+
+	// Test checkID is working
+	assert.NoError(t, bic.checkID(chunkNumber, got))
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
+	assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
+	assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
+	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
+}
+
+func (f *Fs) testFeatures(t *testing.T) {
+	// Check first feature flags are set on this remote
 	enabled := f.Features().SetTier
 	assert.True(t, enabled)
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+type ReadSeekCloser struct {
+	*strings.Reader
+}
+
+func (r *ReadSeekCloser) Close() error {
+	return nil
+}
+
+// Stage a block at remote but don't commit it
+func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
+	var (
+		containerName, blobPath = f.split(remote)
+		containerClient         = f.cntSVC(containerName)
+		blobClient              = containerClient.NewBlockBlobClient(blobPath)
+		data                    = "uncommitted data"
+		blockID                 = "1"
+		blockIDBase64           = base64.StdEncoding.EncodeToString([]byte(blockID))
+	)
+	r := &ReadSeekCloser{strings.NewReader(data)}
+	_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
+	require.NoError(t, err)
+
+	// Verify the block is staged but not committed
+	blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
+	require.NoError(t, err)
+	found := false
+	for _, block := range blockList.UncommittedBlocks {
+		if *block.Name == blockIDBase64 {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "Block ID not found in uncommitted blocks")
+}
+
+// This tests uploading a blob where it has uncommitted blocks with a different ID size.
+//
+// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
+//
+// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
+func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
+	var (
+		ctx    = context.Background()
+		remote = "testBlob"
+	)
+
+	// Multipart copy the blob please
+	oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
+	f.opt.UseCopyBlob = false
+	f.opt.CopyCutoff = f.opt.ChunkSize
+	defer func() {
+		f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
+	}()
+
+	// Create a blob with uncommitted blocks
+	f.stageBlockWithoutCommit(ctx, t, remote)
+
+	// Now attempt to overwrite the block with a different sized block ID to provoke this error
+
+	// Check the object does not exist
+	_, err := f.NewObject(ctx, remote)
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// Upload a multipart file over the block with uncommitted chunks of a different ID size
+	size := 4*int(f.opt.ChunkSize) - 1
+	contents := random.String(size)
+	item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
+	o := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+
+	// Check size
+	assert.Equal(t, int64(size), o.Size())
+
+	// Create a new blob with uncommitted blocks
+	newRemote := "testBlob2"
+	f.stageBlockWithoutCommit(ctx, t, newRemote)
+
+	// Copy over that block
+	dst, err := f.Copy(ctx, o, newRemote)
+	require.NoError(t, err)
+
+	// Check basics
+	assert.Equal(t, int64(size), dst.Size())
+	assert.Equal(t, newRemote, dst.Remote())
+
+	// Check contents
+	gotContents := fstests.ReadObject(ctx, t, dst, -1)
+	assert.Equal(t, contents, gotContents)
+
+	// Remove the object
+	require.NoError(t, dst.Remove(ctx))
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Features", f.testFeatures)
+	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
+}

backend/azureblob/azureblob_test.go

@@ -15,13 +15,17 @@ import (
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
+	name := "TestAzureBlob"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
+		RemoteName:  name + ":",
 		NilObject:   (*Object)(nil),
 		TiersToTest: []string{"Hot", "Cool", "Cold"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: defaultChunkSize,
 		},
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "use_copy_blob", Value: "false"},
+		},
 	})
 }
 
@@ -40,6 +44,7 @@ func TestIntegration2(t *testing.T) {
 		},
 		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "directory_markers", Value: "true"},
+			{Name: name, Key: "use_copy_blob", Value: "false"},
 		},
 	})
 }
@@ -48,8 +53,13 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }
 
+func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setCopyCutoff(cs)
+}
+
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )
 
 func TestValidateAccessTier(t *testing.T) {

backend/azurefiles/azurefiles.go

@@ -237,6 +237,30 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Help:      "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
 			Advanced:  true,
 			Sensitive: true,
+		}, {
+			Name: "disable_instance_discovery",
+			Help: `Skip requesting Microsoft Entra instance metadata
+This should be set true only by applications authenticating in
+disconnected clouds, or private clouds such as Azure Stack.
+It determines whether rclone requests Microsoft Entra instance
+metadata from ` + "`https://login.microsoft.com/`" + ` before
+authenticating.
+Setting this to true will skip this request, making you responsible
+for ensuring the configured authority is valid and trustworthy.
+`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "use_az",
+			Help: `Use Azure CLI tool az for authentication
+Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
+as the sole means of authentication.
+Setting this can be useful if you wish to use the az CLI on a host with
+a System Managed Identity that you do not want to use.
+Don't set env_auth at the same time.
+`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -319,10 +343,12 @@ type Options struct {
 	Username                   string               `config:"username"`
 	Password                   string               `config:"password"`
 	ServicePrincipalFile       string               `config:"service_principal_file"`
+	DisableInstanceDiscovery   bool                 `config:"disable_instance_discovery"`
 	UseMSI                     bool                 `config:"use_msi"`
 	MSIObjectID                string               `config:"msi_object_id"`
 	MSIClientID                string               `config:"msi_client_id"`
 	MSIResourceID              string               `config:"msi_mi_res_id"`
+	UseAZ                      bool                 `config:"use_az"`
 	Endpoint                   string               `config:"endpoint"`
 	ChunkSize                  fs.SizeSuffix        `config:"chunk_size"`
 	MaxStreamSize              fs.SizeSuffix        `config:"max_stream_size"`
@@ -414,7 +440,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		}
 		// Read credentials from the environment
 		options := azidentity.DefaultAzureCredentialOptions{
-			ClientOptions: policyClientOptions,
+			ClientOptions:            policyClientOptions,
+			DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
 		}
 		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
@@ -425,6 +452,13 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		if err != nil {
 			return nil, fmt.Errorf("create new shared key credential failed: %w", err)
 		}
+	case opt.UseAZ:
+		var options = azidentity.AzureCLICredentialOptions{}
+		cred, err = azidentity.NewAzureCLICredential(&options)
+		fmt.Println(cred)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
+		}
 	case opt.SASURL != "":
 		client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt)
 		if err != nil {
@@ -899,7 +933,7 @@ func (o *Object) getMetadata(ctx context.Context) error {
 
 // Hash returns the MD5 of an object returning a lowercase hex string
 //
-// May make a network request becaue the [fs.List] method does not
+// May make a network request because the [fs.List] method does not
 // return MD5 hashes for DirEntry
 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
 	if ty != hash.MD5 {

backend/azurefiles/azurefiles_internal_test.go

@@ -61,7 +61,7 @@ const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
 
 func randomString(charCount int) string {
 	strBldr := strings.Builder{}
-	for i := 0; i < charCount; i++ {
+	for range charCount {
 		randPos := rand.Int63n(52)
 		strBldr.WriteByte(chars[randPos])
 	}

backend/b2/api/types.go

@@ -130,10 +130,10 @@ type AuthorizeAccountResponse struct {
 	AbsoluteMinimumPartSize int      `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
 	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
-		BucketID     string      `json:"bucketId"`     // When present, access is restricted to one bucket.
-		BucketName   string      `json:"bucketName"`   // When present, name of bucket - may be empty
-		Capabilities []string    `json:"capabilities"` // A list of strings, each one naming a capability the key has.
-		NamePrefix   interface{} `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
+		BucketID     string   `json:"bucketId"`     // When present, access is restricted to one bucket.
+		BucketName   string   `json:"bucketName"`   // When present, name of bucket - may be empty
+		Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
+		NamePrefix   any      `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`
 	APIURL              string `json:"apiUrl"`              // The base URL to use for all API calls except for uploading and downloading files.
 	AuthorizationToken  string `json:"authorizationToken"`  // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.

backend/b2/b2.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"net/http"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -30,6 +31,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
@@ -299,14 +301,13 @@ type Fs struct {
 
 // Object describes a b2 object
 type Object struct {
-	fs       *Fs               // what this object is part of
-	remote   string            // The remote path
-	id       string            // b2 id of the file
-	modTime  time.Time         // The modified time of the object if known
-	sha1     string            // SHA-1 hash if known
-	size     int64             // Size of the object
-	mimeType string            // Content-Type of the object
-	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
+	fs       *Fs       // what this object is part of
+	remote   string    // The remote path
+	id       string    // b2 id of the file
+	modTime  time.Time // The modified time of the object if known
+	sha1     string    // SHA-1 hash if known
+	size     int64     // Size of the object
+	mimeType string    // Content-Type of the object
 }
 
 // ------------------------------------------------------------
@@ -589,12 +590,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 
 // hasPermission returns if the current AuthorizationToken has the selected permission
 func (f *Fs) hasPermission(permission string) bool {
-	for _, capability := range f.info.Allowed.Capabilities {
-		if capability == permission {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(f.info.Allowed.Capabilities, permission)
 }
 
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
@@ -1275,7 +1271,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 	toBeDeleted := make(chan *api.File, f.ci.Transfers)
 	var wg sync.WaitGroup
 	wg.Add(f.ci.Transfers)
-	for i := 0; i < f.ci.Transfers; i++ {
+	for range f.ci.Transfers {
 		go func() {
 			defer wg.Done()
 			for object := range toBeDeleted {
@@ -1318,16 +1314,22 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 				// Check current version of the file
 				if deleteHidden && object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove hide marker") {
+						toBeDeleted <- object
+					}
 				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove pending upload") {
+						toBeDeleted <- object
+					}
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
-				toBeDeleted <- object
+				if !operations.SkipDestructive(ctx, object.Name, "delete") {
+					toBeDeleted <- object
+				}
 			}
 			last = remote
 			tr.Done(ctx, nil)
@@ -1598,9 +1600,6 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if err != nil {
 		return err
 	}
-	// For now, just set "mtime" in metadata
-	o.meta = make(map[string]string, 1)
-	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
 	return nil
 }
 
@@ -1880,13 +1879,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		Info:            Info,
 	}
 
-	// Embryonic metadata support - just mtime
-	o.meta = make(map[string]string, 1)
-	modTime, err := parseTimeStringHelper(info.Info[timeKey])
-	if err == nil {
-		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
-	}
-
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -1943,7 +1935,7 @@ func init() {
 // urlEncode encodes in with % encoding
 func urlEncode(in string) string {
 	var out bytes.Buffer
-	for i := 0; i < len(in); i++ {
+	for i := range len(in) {
 		c := in[i]
 		if noNeedToEncode[c] {
 			_ = out.WriteByte(c)
@@ -2264,7 +2256,7 @@ See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 	},
 }
 
-func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	var newRule api.LifecycleRule
 	if daysStr := opt["daysFromHidingToDeleting"]; daysStr != "" {
 		days, err := strconv.Atoi(daysStr)
@@ -2293,8 +2285,10 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 
 	}
 
+	skip := operations.SkipDestructive(ctx, name, "update lifecycle rules")
+
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if !skip && (newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil) {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err
@@ -2351,7 +2345,7 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
 	},
 }
 
-func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	maxAge := defaultMaxAge
 	if opt["max-age"] != "" {
 		maxAge, err = fs.ParseDuration(opt["max-age"])
@@ -2374,7 +2368,7 @@ it would do.
 `,
 }
 
-func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	return nil, f.cleanUp(ctx, true, false, 0)
 }
 
@@ -2393,7 +2387,7 @@ var commandHelp = []fs.CommandHelp{
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "lifecycle":
 		return f.lifecycleCommand(ctx, name, arg, opt)

backend/b2/b2_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -13,6 +14,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/bucket"
@@ -256,12 +258,6 @@ func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string
 			assert.Equal(t, v, got, k)
 		}
 
-		// mtime
-		for k, v := range metadata {
-			got := o.meta[k]
-			assert.Equal(t, v, got, k)
-		}
-
 		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
 
 		// Modification time from the x-bz-info-src_last_modified_millis header
@@ -463,24 +459,161 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	})
 
 	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
+		t.Run("DryRun", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should be unchanged after dry run
+			before := listAllFiles(ctx, t, f, dirName)
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			after := listAllFiles(ctx, t, f, dirName)
+			assert.Equal(t, before, after)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should reflect current state after cleanup
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+			fstest.CheckListing(t, f, items)
+		})
 	})
 
 	// Purge gets tested later
 }
 
+func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
+	ctx := context.Background()
+
+	// B2CleanupHidden tests cleaning up hidden files
+	t.Run("CleanupUnfinished", func(t *testing.T) {
+		dirName := "unfinished"
+		fileCount := 5
+		expectedFiles := []string{}
+		for i := 1; i < fileCount; i++ {
+			fileName := fmt.Sprintf("%s/unfinished-%d", dirName, i)
+			expectedFiles = append(expectedFiles, fileName)
+			obj := &Object{
+				fs:     f,
+				remote: fileName,
+			}
+			objInfo := object.NewStaticObjectInfo(fileName, fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
+			_, err := f.newLargeUpload(ctx, obj, nil, objInfo, f.opt.ChunkSize, false, nil)
+			require.NoError(t, err)
+		}
+		checkListing(ctx, t, f, dirName, expectedFiles)
+
+		t.Run("DryRun", func(t *testing.T) {
+			// Listing should not change after dry run
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, expectedFiles)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			// Listing should be empty after real cleanup
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, []string{})
+		})
+	})
+}
+
+func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
+	bucket, directory := f.split(dirName)
+	foundFiles := []string{}
+	require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
+		if !isDirectory {
+			foundFiles = append(foundFiles, object.Name)
+		}
+		return nil
+	}))
+	sort.Strings(foundFiles)
+	return foundFiles
+}
+
+func checkListing(ctx context.Context, t *testing.T, f *Fs, dirName string, expectedFiles []string) {
+	foundFiles := listAllFiles(ctx, t, f, dirName)
+	sort.Strings(expectedFiles)
+	assert.Equal(t, expectedFiles, foundFiles)
+}
+
+func (f *Fs) InternalTestLifecycleRules(t *testing.T) {
+	ctx := context.Background()
+
+	opt := map[string]string{}
+
+	t.Run("InitState", func(t *testing.T) {
+		// There should be no lifecycle rules at the outset
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("DryRun", func(t *testing.T) {
+		// There should still be no lifecycle rules after each dry run operation
+		ctx, ci := fs.AddConfig(ctx)
+		ci.DryRun = true
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("RealThing", func(t *testing.T) {
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+	})
+}
+
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("Versions", f.InternalTestVersions)
+	t.Run("CleanupUnfinished", f.InternalTestCleanupUnfinished)
+	t.Run("LifecycleRules", f.InternalTestLifecycleRules)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/upload.go

@@ -478,17 +478,14 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
 		remaining = up.size
 	)
 	g.SetLimit(up.f.opt.UploadConcurrency)
-	for part := 0; part < up.parts; part++ {
+	for part := range up.parts {
 		// Fail fast, in case an errgroup managed function returns an error
 		// gCtx is cancelled. There is no point in copying all the other parts.
 		if gCtx.Err() != nil {
 			break
 		}
 
-		reqSize := remaining
-		if reqSize >= up.chunkSize {
-			reqSize = up.chunkSize
-		}
+		reqSize := min(remaining, up.chunkSize)
 
 		part := part // for the closure
 		g.Go(func() (err error) {

backend/box/box.go

@@ -237,8 +237,8 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	return claims, nil
 }
 
-func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
-	signingHeaders := map[string]interface{}{
+func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]any {
+	signingHeaders := map[string]any{
 		"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
 	}
 	return signingHeaders
@@ -1343,12 +1343,8 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 	nextStreamPosition = streamPosition
 
 	for {
-		limit := f.opt.ListChunk
-
 		// box only allows a max of 500 events
-		if limit > 500 {
-			limit = 500
-		}
+		limit := min(f.opt.ListChunk, 500)
 
 		opts := rest.Opts{
 			Method:     "GET",

backend/box/upload.go

@@ -105,7 +105,7 @@ func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api
 	const defaultDelay = 10
 	var tries int
 outer:
-	for tries = 0; tries < maxTries; tries++ {
+	for tries = range maxTries {
 		err = o.fs.pacer.Call(func() (bool, error) {
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
@@ -203,7 +203,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	errs := make(chan error, 1)
 	var wg sync.WaitGroup
 outer:
-	for part := 0; part < session.TotalParts; part++ {
+	for part := range session.TotalParts {
 		// Check any errors
 		select {
 		case err = <-errs:
@@ -211,10 +211,7 @@ outer:
 		default:
 		}
 
-		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
-		}
+		reqSize := min(remaining, chunkSize)
 
 		// Make a block of memory
 		buf := make([]byte, reqSize)

backend/cache/cache.go

@@ -1092,7 +1092,7 @@ func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) er
 		return err
 	}
 
-	for i := 0; i < len(entries); i++ {
+	for i := range entries {
 		innerDir, ok := entries[i].(fs.Directory)
 		if ok {
 			err := f.recurse(ctx, innerDir.Remote(), list)
@@ -1428,7 +1428,7 @@ func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn i
 	}()
 
 	// wait until both are done
-	for c := 0; c < 2; c++ {
+	for range 2 {
 		<-done
 	}
 }
@@ -1753,7 +1753,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 }
 
 // Stats returns stats about the cache storage
-func (f *Fs) Stats() (map[string]map[string]interface{}, error) {
+func (f *Fs) Stats() (map[string]map[string]any, error) {
 	return f.cache.Stats()
 }
 
@@ -1933,7 +1933,7 @@ var commandHelp = []fs.CommandHelp{
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
 	switch name {
 	case "stats":
 		return f.Stats()

backend/cache/cache_internal_test.go

@@ -360,7 +360,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
-	for i := 0; i < len(checkSample); i++ {
+	for i := range checkSample {
 		require.Equal(t, testData[i], checkSample[i])
 	}
 }
@@ -387,7 +387,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 	readData, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
 	require.NoError(t, err)
-	for i := 0; i < len(readData); i++ {
+	for i := range readData {
 		require.Equalf(t, testData[i], readData[i], "at byte %v", i)
 	}
 }
@@ -688,7 +688,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	co, ok := o.(*cache.Object)
 	require.True(t, ok)
 
-	for i := 0; i < 4; i++ { // read first 4
+	for i := range 4 { // read first 4
 		_ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false)
 	}
 	cfs.CleanUpCache(true)
@@ -971,7 +971,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
-	for i := 0; i < int(cnt); i++ {
+	for range int(cnt) {
 		data := randStringBytes(int(chunk))
 		_, _ = f.Write(data)
 	}
@@ -1085,9 +1085,9 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 	return err
 }
 
-func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) {
+func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]any, error) {
 	var err error
-	var l []interface{}
+	var l []any
 	var list fs.DirEntries
 	list, err = f.List(context.Background(), remote)
 	for _, ll := range list {
@@ -1215,7 +1215,7 @@ func (r *run) listenForBackgroundUpload(t *testing.T, f fs.Fs, remote string) ch
 		var err error
 		var state cache.BackgroundUploadState
 
-		for i := 0; i < 2; i++ {
+		for range 2 {
 			select {
 			case state = <-buCh:
 				// continue
@@ -1293,7 +1293,7 @@ func (r *run) completeAllBackgroundUploads(t *testing.T, f fs.Fs, lastRemote str
 
 func (r *run) retryBlock(block func() error, maxRetries int, rate time.Duration) error {
 	var err error
-	for i := 0; i < maxRetries; i++ {
+	for range maxRetries {
 		err = block()
 		if err == nil {
 			return nil

backend/cache/cache_upload_test.go

@@ -162,7 +162,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
 
 	lastFile := ""
-	for i := 0; i < totalFiles; i++ {
+	for i := range totalFiles {
 		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"

backend/cache/handle.go

@@ -182,7 +182,7 @@ func (r *Handle) queueOffset(offset int64) {
 			}
 		}
 
-		for i := 0; i < r.workers; i++ {
+		for i := range r.workers {
 			o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
 			if o < 0 || o >= r.cachedObject.Size() {
 				continue
@@ -222,7 +222,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	if !found {
 		// we're gonna give the workers a chance to pickup the chunk
 		// and retry a couple of times
-		for i := 0; i < r.cacheFs().opt.ReadRetries*8; i++ {
+		for i := range r.cacheFs().opt.ReadRetries * 8 {
 			data, err = r.storage().GetChunk(r.cachedObject, chunkStart)
 			if err == nil {
 				found = true

backend/cache/plex.go

@@ -209,7 +209,7 @@ func (p *plexConnector) authenticate() error {
 	if err != nil {
 		return err
 	}
-	var data map[string]interface{}
+	var data map[string]any
 	err = json.NewDecoder(resp.Body).Decode(&data)
 	if err != nil {
 		return fmt.Errorf("failed to obtain token: %w", err)
@@ -273,11 +273,11 @@ func (p *plexConnector) isPlaying(co *Object) bool {
 }
 
 // adapted from: https://stackoverflow.com/a/28878037 (credit)
-func get(m interface{}, path ...interface{}) (interface{}, bool) {
+func get(m any, path ...any) (any, bool) {
 	for _, p := range path {
 		switch idx := p.(type) {
 		case string:
-			if mm, ok := m.(map[string]interface{}); ok {
+			if mm, ok := m.(map[string]any); ok {
 				if val, found := mm[idx]; found {
 					m = val
 					continue
@@ -285,7 +285,7 @@ func get(m interface{}, path ...interface{}) (interface{}, bool) {
 			}
 			return nil, false
 		case int:
-			if mm, ok := m.([]interface{}); ok {
+			if mm, ok := m.([]any); ok {
 				if len(mm) > idx {
 					m = mm[idx]
 					continue

backend/cache/storage_persistent.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
 	bolt "go.etcd.io/bbolt"
+	"go.etcd.io/bbolt/errors"
 )
 
 // Constants
@@ -597,7 +598,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 	})
 
 	if err != nil {
-		if err == bolt.ErrDatabaseNotOpen {
+		if err == errors.ErrDatabaseNotOpen {
 			// we're likely a late janitor and we need to end quietly as there's no guarantee of what exists anymore
 			return
 		}
@@ -606,16 +607,16 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 }
 
 // Stats returns a go map with the stats key values
-func (b *Persistent) Stats() (map[string]map[string]interface{}, error) {
-	r := make(map[string]map[string]interface{})
-	r["data"] = make(map[string]interface{})
+func (b *Persistent) Stats() (map[string]map[string]any, error) {
+	r := make(map[string]map[string]any)
+	r["data"] = make(map[string]any)
 	r["data"]["oldest-ts"] = time.Now()
 	r["data"]["oldest-file"] = ""
 	r["data"]["newest-ts"] = time.Now()
 	r["data"]["newest-file"] = ""
 	r["data"]["total-chunks"] = 0
 	r["data"]["total-size"] = int64(0)
-	r["files"] = make(map[string]interface{})
+	r["files"] = make(map[string]any)
 	r["files"]["oldest-ts"] = time.Now()
 	r["files"]["oldest-name"] = ""
 	r["files"]["newest-ts"] = time.Now()

backend/chunker/chunker.go

@@ -632,7 +632,7 @@ func (f *Fs) parseChunkName(filePath string) (parentPath string, chunkNo int, ct
 
 // forbidChunk prints error message or raises error if file is chunk.
 // First argument sets log prefix, use `false` to suppress message.
-func (f *Fs) forbidChunk(o interface{}, filePath string) error {
+func (f *Fs) forbidChunk(o any, filePath string) error {
 	if parentPath, _, _, _ := f.parseChunkName(filePath); parentPath != "" {
 		if f.opt.FailHard {
 			return fmt.Errorf("chunk overlap with %q", parentPath)
@@ -680,7 +680,7 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
 	circleSec := unixSec % closestPrimeZzzzSeconds
 	first4chars := strconv.FormatInt(circleSec, 36)
 
-	for tries := 0; tries < maxTransactionProbes; tries++ {
+	for range maxTransactionProbes {
 		f.xactIDMutex.Lock()
 		randomness := f.xactIDRand.Int63n(maxTwoBase36Digits + 1)
 		f.xactIDMutex.Unlock()
@@ -1189,10 +1189,7 @@ func (f *Fs) put(
 		}
 
 		tempRemote := f.makeChunkName(baseRemote, c.chunkNo, "", xactID)
-		size := c.sizeLeft
-		if size > c.chunkSize {
-			size = c.chunkSize
-		}
+		size := min(c.sizeLeft, c.chunkSize)
 		savedReadCount := c.readCount
 
 		// If a single chunk is expected, avoid the extra rename operation
@@ -1477,10 +1474,7 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
 	const bufLen = 1048576 // 1 MiB
 	buf := make([]byte, bufLen)
 	for size > 0 {
-		n := size
-		if n > bufLen {
-			n = bufLen
-		}
+		n := min(size, bufLen)
 		if _, err := io.ReadFull(in, buf[0:n]); err != nil {
 			return err
 		}
@@ -2480,7 +2474,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	if len(data) > maxMetadataSizeWritten {
 		return nil, false, ErrMetaTooBig
 	}
-	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
+	if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
 		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON

backend/chunker/chunker_internal_test.go

@@ -40,7 +40,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
 	})
 }
 
-type settings map[string]interface{}
+type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
 	fsName := strings.Split(f.Name(), "{")[0] // strip off hash

backend/crypt/cipher.go

@@ -192,7 +192,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 		dirNameEncrypt:  dirNameEncrypt,
 		encryptedSuffix: ".bin",
 	}
-	c.buffers.New = func() interface{} {
+	c.buffers.New = func() any {
 		return new([blockSize]byte)
 	}
 	err := c.Key(password, salt)
@@ -336,7 +336,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	_, _ = result.WriteString(strconv.Itoa(dir) + ".")
 
 	// but we'll augment it with the nameKey for real calculation
-	for i := 0; i < len(c.nameKey); i++ {
+	for i := range len(c.nameKey) {
 		dir += int(c.nameKey[i])
 	}
 
@@ -418,7 +418,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	}
 
 	// add the nameKey to get the real rotate distance
-	for i := 0; i < len(c.nameKey); i++ {
+	for i := range len(c.nameKey) {
 		dir += int(c.nameKey[i])
 	}
 
@@ -664,7 +664,7 @@ func (n *nonce) increment() {
 // add a uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
-	for i := 0; i < 8; i++ {
+	for i := range 8 {
 		digit := (*n)[i]
 		xDigit := byte(x)
 		x >>= 8

backend/crypt/cipher_test.go

@@ -1307,10 +1307,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 	open := func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
 		end := len(ciphertext)
 		if underlyingLimit >= 0 {
-			end = int(underlyingOffset + underlyingLimit)
-			if end > len(ciphertext) {
-				end = len(ciphertext)
-			}
+			end = min(int(underlyingOffset+underlyingLimit), len(ciphertext))
 		}
 		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
@@ -1490,7 +1487,7 @@ func TestDecrypterRead(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Test truncating the file at each possible point
-	for i := 0; i < len(file16)-1; i++ {
+	for i := range len(file16) - 1 {
 		what := fmt.Sprintf("truncating to %d/%d", i, len(file16))
 		cd := newCloseDetector(bytes.NewBuffer(file16[:i]))
 		fh, err := c.newDecrypter(cd)

backend/crypt/crypt.go

@@ -924,7 +924,7 @@ Usage Example:
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "decode":
 		out := make([]string, 0, len(arg))

backend/crypt/pkcs7/pkcs7.go

@@ -25,7 +25,7 @@ func Pad(n int, buf []byte) []byte {
 	}
 	length := len(buf)
 	padding := n - (length % n)
-	for i := 0; i < padding; i++ {
+	for range padding {
 		buf = append(buf, byte(padding))
 	}
 	if (len(buf) % n) != 0 {
@@ -54,7 +54,7 @@ func Unpad(n int, buf []byte) ([]byte, error) {
 	if padding == 0 {
 		return nil, ErrorPaddingTooShort
 	}
-	for i := 0; i < padding; i++ {
+	for i := range padding {
 		if buf[length-1-i] != byte(padding) {
 			return nil, ErrorPaddingNotAllTheSame
 		}

backend/drive/drive.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -199,13 +200,7 @@ func driveScopes(scopesString string) (scopes []string) {
 
 // Returns true if one of the scopes was "drive.appfolder"
 func driveScopesContainsAppFolder(scopes []string) bool {
-	for _, scope := range scopes {
-		if scope == scopePrefix+"drive.appfolder" {
-			return true
-		}
-
-	}
-	return false
+	return slices.Contains(scopes, scopePrefix+"drive.appfolder")
 }
 
 func driveOAuthOptions() []fs.Option {
@@ -959,12 +954,7 @@ func parseDrivePath(path string) (root string, err error) {
 type listFn func(*drive.File) bool
 
 func containsString(slice []string, s string) bool {
-	for _, e := range slice {
-		if e == s {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(slice, s)
 }
 
 // getFile returns drive.File for the ID passed and fields passed in
@@ -1153,13 +1143,7 @@ OUTER:
 			// Check the case of items is correct since
 			// the `=` operator is case insensitive.
 			if title != "" && title != item.Name {
-				found := false
-				for _, stem := range stems {
-					if stem == item.Name {
-						found = true
-						break
-					}
-				}
+				found := slices.Contains(stems, item.Name)
 				if !found {
 					continue
 				}
@@ -1212,6 +1196,7 @@ func fixMimeType(mimeTypeIn string) string {
 	}
 	return mimeTypeOut
 }
+
 func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	out = make(map[string][]string, len(in))
 	for k, v := range in {
@@ -1222,9 +1207,11 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	}
 	return out
 }
+
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
 }
+
 func isLinkMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/x-link-")
 }
@@ -1559,13 +1546,10 @@ func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
 func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
 	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
 	if f.opt.SkipChecksumGphotos {
-		for _, space := range info.Spaces {
-			if space == "photos" {
-				info.Md5Checksum = ""
-				info.Sha1Checksum = ""
-				info.Sha256Checksum = ""
-				break
-			}
+		if slices.Contains(info.Spaces, "photos") {
+			info.Md5Checksum = ""
+			info.Sha1Checksum = ""
+			info.Sha256Checksum = ""
 		}
 	}
 	o := &Object{
@@ -1657,7 +1641,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	ctx context.Context, remote string, info *drive.File,
-	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
+	extension, exportName, exportMimeType string, isDocument bool,
+) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
@@ -1848,6 +1833,7 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
+
 func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
@@ -1893,7 +1879,8 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
 func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
-	extension, mimeType string, isDocument bool) {
+	extension, mimeType string, isDocument bool,
+) {
 	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
@@ -2240,7 +2227,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	wg.Add(1)
 	in <- listREntry{directoryID, dir}
 
-	for i := 0; i < f.ci.Checkers; i++ {
+	for range f.ci.Checkers {
 		go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
 	}
 	go func() {
@@ -2249,11 +2236,8 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		// if the input channel overflowed add the collected entries to the channel now
 		for len(overflow) > 0 {
 			mu.Lock()
-			l := len(overflow)
 			// only fill half of the channel to prevent entries being put into overflow again
-			if l > listRInputBuffer/2 {
-				l = listRInputBuffer / 2
-			}
+			l := min(len(overflow), listRInputBuffer/2)
 			wg.Add(l)
 			for _, d := range overflow[:l] {
 				in <- d
@@ -2273,7 +2257,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		mu.Unlock()
 	}()
 	// wait until the all workers to finish
-	for i := 0; i < f.ci.Checkers; i++ {
+	for range f.ci.Checkers {
 		e := <-out
 		mu.Lock()
 		// if one worker returns an error early, close the input so all other workers exit
@@ -2689,7 +2673,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if shortcutID != "" {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
-	var trashedFiles = false
+	trashedFiles := false
 	if check {
 		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
 			if !item.Trashed {
@@ -2926,7 +2910,6 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
-
 	if err != nil {
 		return err
 	}
@@ -3187,6 +3170,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
+
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
@@ -3525,14 +3509,14 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 	return f.unTrash(ctx, dir, directoryID, true)
 }
 
-// copy file with id to dest
-func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
+// copy or move file with id to dest
+func (f *Fs) copyOrMoveID(ctx context.Context, operation string, id, dest string) (err error) {
 	info, err := f.getFile(ctx, id, f.getFileFields(ctx))
 	if err != nil {
 		return fmt.Errorf("couldn't find id: %w", err)
 	}
 	if info.MimeType == driveFolderType {
-		return fmt.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
+		return fmt.Errorf("can't %s directory use: rclone %s --drive-root-folder-id %s %s %s", operation, operation, id, fs.ConfigString(f), dest)
 	}
 	info.Name = f.opt.Enc.ToStandardName(info.Name)
 	o, err := f.newObjectWithInfo(ctx, info.Name, info)
@@ -3553,9 +3537,15 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = operations.Copy(ctx, dstFs, nil, destLeaf, o)
-	if err != nil {
-		return fmt.Errorf("copy failed: %w", err)
+
+	var opErr error
+	if operation == "moveid" {
+		_, opErr = operations.Move(ctx, dstFs, nil, destLeaf, o)
+	} else {
+		_, opErr = operations.Copy(ctx, dstFs, nil, destLeaf, o)
+	}
+	if opErr != nil {
+		return fmt.Errorf("%s failed: %w", operation, opErr)
 	}
 	return nil
 }
@@ -3792,6 +3782,28 @@ attempted if possible.
 
 Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
 `,
+}, {
+	Name:  "moveid",
+	Short: "Move files by ID",
+	Long: `This command moves files by ID
+
+Usage:
+
+    rclone backend moveid drive: ID path
+    rclone backend moveid drive: ID1 path1 ID2 path2
+
+It moves the drive file with ID given to the path (an rclone path which
+will be passed internally to rclone moveto).
+
+The path should end with a / to indicate move the file as named to
+this directory. If it doesn't end with a / then the last path
+component will be used as the file name.
+
+If the destination is a drive backend then server-side moving will be
+attempted if possible.
+
+Use the --interactive/-i or --dry-run flag to see what would be moved beforehand.
+`,
 }, {
 	Name:  "exportformats",
 	Short: "Dump the export formats for debug purposes",
@@ -3881,7 +3893,7 @@ Third delete all orphaned files to the trash
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "get":
 		out := make(map[string]string)
@@ -3970,16 +3982,16 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			dir = arg[0]
 		}
 		return f.unTrashDir(ctx, dir, true)
-	case "copyid":
+	case "copyid", "moveid":
 		if len(arg)%2 != 0 {
 			return nil, errors.New("need an even number of arguments")
 		}
 		for len(arg) > 0 {
 			id, dest := arg[0], arg[1]
 			arg = arg[2:]
-			err = f.copyID(ctx, id, dest)
+			err = f.copyOrMoveID(ctx, name, id, dest)
 			if err != nil {
-				return nil, fmt.Errorf("failed copying %q to %q: %w", id, dest, err)
+				return nil, fmt.Errorf("failed %s %q to %q: %w", name, id, dest, err)
 			}
 		}
 		return nil, nil
@@ -3990,14 +4002,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	case "query":
 		if len(arg) == 1 {
 			query := arg[0]
-			var results, err = f.query(ctx, query)
+			results, err := f.query(ctx, query)
 			if err != nil {
 				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
 			}
 			return results, nil
-		} else {
-			return nil, errors.New("need a query argument")
 		}
+		return nil, errors.New("need a query argument")
 	case "rescue":
 		dirID := ""
 		_, delete := opt["delete"]
@@ -4057,6 +4068,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	return "", hash.ErrUnsupported
 }
+
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
 		return "", hash.ErrUnsupported
@@ -4071,7 +4083,8 @@ func (o *baseObject) Size() int64 {
 
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
-	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
+	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error,
+) {
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -4284,12 +4297,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	return o.baseObject.open(ctx, o.url, options...)
 }
+
 func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
 	var offset, end int64 = 0, -1
-	var newOptions = options[:0]
+	newOptions := options[:0]
 	for _, o := range options {
 		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
@@ -4316,9 +4330,10 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
 	}
 	return
 }
+
 func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
-	var data = o.content
+	data := o.content
 	for _, option := range options {
 		switch x := option.(type) {
 		case *fs.SeekOption:
@@ -4343,7 +4358,8 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
-	src fs.ObjectInfo) (info *drive.File, err error) {
+	src fs.ObjectInfo,
+) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
 	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
@@ -4421,6 +4437,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	return nil
 }
+
 func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	srcMimeType := fs.MimeType(ctx, src)
 	importMimeType := ""
@@ -4516,6 +4533,7 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
+
 func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }

backend/drive/drive_internal_test.go

@@ -479,8 +479,8 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) {
 	require.NoError(t, f.Purge(ctx, "trashDir"))
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
+// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyOrMoveID
+func (f *Fs) InternalTestCopyOrMoveID(t *testing.T) {
 	ctx := context.Background()
 	obj, err := f.NewObject(ctx, existingFile)
 	require.NoError(t, err)
@@ -498,7 +498,7 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	}
 
 	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", "ID-NOT-FOUND", dir+"/")
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "couldn't find id")
 	})
@@ -506,19 +506,31 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	t.Run("Directory", func(t *testing.T) {
 		rootID, err := f.dirCache.RootID(ctx, false)
 		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
+		err = f.copyOrMoveID(ctx, "moveid", rootID, dir+"/")
 		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
+		assert.Contains(t, err.Error(), "can't moveid directory")
 	})
 
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
+	t.Run("MoveWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/")
 		require.NoError(t, err)
 		checkFile(path.Base(existingFile))
 	})
 
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
+	t.Run("CopyWithoutDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/")
+		require.NoError(t, err)
+		checkFile(path.Base(existingFile))
+	})
+
+	t.Run("MoveWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/potato.txt")
+		require.NoError(t, err)
+		checkFile("potato.txt")
+	})
+
+	t.Run("CopyWithDestName", func(t *testing.T) {
+		err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/potato.txt")
 		require.NoError(t, err)
 		checkFile("potato.txt")
 	})
@@ -647,7 +659,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	})
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
+	t.Run("CopyOrMoveID", f.InternalTestCopyOrMoveID)
 	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)

backend/drive/metadata.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"strconv"
 	"strings"
 	"sync"
@@ -324,9 +325,7 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
 	metadata := make(fs.Metadata, 16)
 
 	// Dump user metadata first as it overrides system metadata
-	for k, v := range info.Properties {
-		metadata[k] = v
-	}
+	maps.Copy(metadata, info.Properties)
 
 	// System metadata
 	metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)

backend/drive/upload.go

@@ -177,10 +177,7 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 			if start >= rx.ContentLength {
 				break
 			}
-			reqSize = rx.ContentLength - start
-			if reqSize >= int64(rx.f.opt.ChunkSize) {
-				reqSize = int64(rx.f.opt.ChunkSize)
-			}
+			reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
 			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 		} else {
 			// If size unknown read into buffer

backend/dropbox/dbhash/dbhash.go

@@ -55,10 +55,7 @@ func (d *digest) Write(p []byte) (n int, err error) {
 	n = len(p)
 	for len(p) > 0 {
 		d.writtenMore = true
-		toWrite := bytesPerBlock - d.n
-		if toWrite > len(p) {
-			toWrite = len(p)
-		}
+		toWrite := min(bytesPerBlock-d.n, len(p))
 		_, err = d.blockHash.Write(p[:toWrite])
 		if err != nil {
 			panic(hashReturnedError)

backend/dropbox/dbhash/dbhash_test.go

@@ -11,7 +11,7 @@ import (
 
 func testChunk(t *testing.T, chunk int) {
 	data := make([]byte, chunk)
-	for i := 0; i < chunk; i++ {
+	for i := range chunk {
 		data[i] = 'A'
 	}
 	for _, test := range []struct {

backend/dropbox/dropbox.go

@@ -92,6 +92,9 @@ const (
 	maxFileNameLength = 255
 )
 
+type exportAPIFormat string
+type exportExtension string // dotless
+
 var (
 	// Description of how to auth for this app
 	dropboxConfig = &oauthutil.Config{
@@ -132,6 +135,16 @@ var (
 		DefaultTimeoutAsync:   10 * time.Second,
 		DefaultBatchSizeAsync: 100,
 	}
+
+	exportKnownAPIFormats = map[exportAPIFormat]exportExtension{
+		"markdown": "md",
+		"html":     "html",
+	}
+	// Populated based on exportKnownAPIFormats
+	exportKnownExtensions = map[exportExtension]exportAPIFormat{}
+
+	paperExtension         = ".paper"
+	paperTemplateExtension = ".papert"
 )
 
 // Gets an oauth config with the right scopes
@@ -247,23 +260,61 @@ folders.`,
 			Help:     "Specify a different Dropbox namespace ID to use as the root for all paths.",
 			Default:  "",
 			Advanced: true,
-		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
+		}, {
+			Name: "export_formats",
+			Help: `Comma separated list of preferred formats for exporting files
+
+Certain Dropbox files can only be accessed by exporting them to another format.
+These include Dropbox Paper documents.
+
+For each such file, rclone will choose the first format on this list that Dropbox
+considers valid. If none is valid, it will choose Dropbox's default format.
+
+Known formats include: "html", "md" (markdown)`,
+			Default:  fs.CommaSepList{"html", "md"},
+			Advanced: true,
+		}, {
+			Name:     "skip_exports",
+			Help:     "Skip exportable files in all listings.\n\nIf given, exportable files practically become invisible to rclone.",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:    "show_all_exports",
+			Default: false,
+			Help: `Show all exportable files in listings.
+
+Adding this flag will allow all exportable files to be server side copied.
+Note that rclone doesn't add extensions to the exportable file names in this mode.
+
+Do **not** use this flag when trying to download exportable files - rclone
+will fail to download them.
+`,
+			Advanced: true,
+		},
+		}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
 	})
+
+	for apiFormat, ext := range exportKnownAPIFormats {
+		exportKnownExtensions[ext] = apiFormat
+	}
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
-	Impersonate   string               `config:"impersonate"`
-	SharedFiles   bool                 `config:"shared_files"`
-	SharedFolders bool                 `config:"shared_folders"`
-	BatchMode     string               `config:"batch_mode"`
-	BatchSize     int                  `config:"batch_size"`
-	BatchTimeout  fs.Duration          `config:"batch_timeout"`
-	AsyncBatch    bool                 `config:"async_batch"`
-	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
-	Enc           encoder.MultiEncoder `config:"encoding"`
-	RootNsid      string               `config:"root_namespace"`
+	ChunkSize      fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate    string               `config:"impersonate"`
+	SharedFiles    bool                 `config:"shared_files"`
+	SharedFolders  bool                 `config:"shared_folders"`
+	BatchMode      string               `config:"batch_mode"`
+	BatchSize      int                  `config:"batch_size"`
+	BatchTimeout   fs.Duration          `config:"batch_timeout"`
+	AsyncBatch     bool                 `config:"async_batch"`
+	PacerMinSleep  fs.Duration          `config:"pacer_min_sleep"`
+	Enc            encoder.MultiEncoder `config:"encoding"`
+	RootNsid       string               `config:"root_namespace"`
+	ExportFormats  fs.CommaSepList      `config:"export_formats"`
+	SkipExports    bool                 `config:"skip_exports"`
+	ShowAllExports bool                 `config:"show_all_exports"`
 }
 
 // Fs represents a remote dropbox server
@@ -283,8 +334,18 @@ type Fs struct {
 	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
+	exportExts     []exportExtension
 }
 
+type exportType int
+
+const (
+	notExport        exportType = iota // a regular file
+	exportHide                         // should be hidden
+	exportListOnly                     // listable, but can't export
+	exportExportable                   // can export
+)
+
 // Object describes a dropbox object
 //
 // Dropbox Objects always have full metadata
@@ -296,6 +357,9 @@ type Object struct {
 	bytes   int64     // size of the object
 	modTime time.Time // time it was last modified
 	hash    string    // content_hash of the object
+
+	exportType      exportType
+	exportAPIFormat exportAPIFormat
 }
 
 // Name of the remote (as passed into NewFs)
@@ -436,6 +500,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		HeaderGenerator: f.headerGenerator,
 	}
 
+	for _, e := range opt.ExportFormats {
+		ext := exportExtension(e)
+		if exportKnownExtensions[ext] == "" {
+			return nil, fmt.Errorf("dropbox: unknown export format '%s'", e)
+		}
+		f.exportExts = append(f.exportExts, ext)
+	}
+
 	// unauthorized config for endpoints that fail with auth
 	ucfg := dropbox.Config{
 		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
@@ -588,38 +660,126 @@ func (f *Fs) setRoot(root string) {
 	}
 }
 
+type getMetadataResult struct {
+	entry    files.IsMetadata
+	notFound bool
+	err      error
+}
+
 // getMetadata gets the metadata for a file or directory
-func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
-	err = f.pacer.Call(func() (bool, error) {
-		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
+func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
+	res.err = f.pacer.Call(func() (bool, error) {
+		res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
-		return shouldRetry(ctx, err)
+		return shouldRetry(ctx, res.err)
 	})
-	if err != nil {
-		switch e := err.(type) {
+	if res.err != nil {
+		switch e := res.err.(type) {
 		case files.GetMetadataAPIError:
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
-				notFound = true
-				err = nil
+				res.notFound = true
+				res.err = nil
 			}
 		}
 	}
 	return
 }
 
-// getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
-	entry, notFound, err := f.getMetadata(ctx, filePath)
-	if err != nil {
-		return nil, err
+// Get metadata such that the result would be exported with the given extension
+// Return a channel that will eventually receive the metadata
+func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportExtension exportExtension) chan getMetadataResult {
+	ch := make(chan getMetadataResult, 1)
+	wantDownloadable := (wantExportExtension == "")
+	go func() {
+		defer close(ch)
+
+		res := f.getMetadata(ctx, filePath)
+		info, ok := res.entry.(*files.FileMetadata)
+		if !ok { // Can't check anything about file, just return what we have
+			ch <- res
+			return
+		}
+
+		// Return notFound if downloadability or extension doesn't match
+		if wantDownloadable != info.IsDownloadable {
+			ch <- getMetadataResult{notFound: true}
+			return
+		}
+		if !info.IsDownloadable {
+			_, ext := f.chooseExportFormat(info)
+			if ext != wantExportExtension {
+				ch <- getMetadataResult{notFound: true}
+				return
+			}
+		}
+
+		// Return our real result or error
+		ch <- res
+	}()
+	return ch
+}
+
+// For a given rclone-path, figure out what the Dropbox-path may be, in order of preference.
+// Multiple paths might be plausible, due to export path munging.
+func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-chan getMetadataResult) {
+	ret = []<-chan getMetadataResult{}
+
+	// Prefer an exact match
+	ret = append(ret, f.getMetadataForExt(ctx, filePath, ""))
+
+	// Check if we're plausibly an export path, otherwise we're done
+	if f.opt.SkipExports || f.opt.ShowAllExports {
+		return
 	}
-	if notFound {
+	dotted := path.Ext(filePath)
+	if dotted == "" {
+		return
+	}
+	ext := exportExtension(dotted[1:])
+	if exportKnownExtensions[ext] == "" {
+		return
+	}
+
+	// We might be an export path! Try all possibilities
+	base := strings.TrimSuffix(filePath, dotted)
+
+	// `foo.papert.md` will only come from `foo.papert`. Never check something like `foo.papert.paper`
+	if strings.HasSuffix(base, paperTemplateExtension) {
+		ret = append(ret, f.getMetadataForExt(ctx, base, ext))
+		return
+	}
+
+	// Otherwise, try both `foo.md` coming from `foo`, or from `foo.paper`
+	ret = append(ret, f.getMetadataForExt(ctx, base, ext))
+	ret = append(ret, f.getMetadataForExt(ctx, base+paperExtension, ext))
+	return
+}
+
+// getFileMetadata gets the metadata for a file
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
+	var res getMetadataResult
+
+	// Try all possible metadatas
+	possibleMetadatas := f.possibleMetadatas(ctx, filePath)
+	for _, ch := range possibleMetadatas {
+		res = <-ch
+
+		if res.err != nil {
+			return nil, res.err
+		}
+		if !res.notFound {
+			break
+		}
+	}
+
+	if res.notFound {
 		return nil, fs.ErrorObjectNotFound
 	}
-	fileInfo, ok := entry.(*files.FileMetadata)
+
+	fileInfo, ok := res.entry.(*files.FileMetadata)
 	if !ok {
-		if _, ok = entry.(*files.FolderMetadata); ok {
+		if _, ok = res.entry.(*files.FolderMetadata); ok {
 			return nil, fs.ErrorIsDir
 		}
 		return nil, fs.ErrorNotAFile
@@ -628,15 +788,15 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *fi
 }
 
 // getDirMetadata gets the metadata for a directory
-func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
-	entry, notFound, err := f.getMetadata(ctx, dirPath)
-	if err != nil {
-		return nil, err
+func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (*files.FolderMetadata, error) {
+	res := f.getMetadata(ctx, dirPath)
+	if res.err != nil {
+		return nil, res.err
 	}
-	if notFound {
+	if res.notFound {
 		return nil, fs.ErrorDirNotFound
 	}
-	dirInfo, ok := entry.(*files.FolderMetadata)
+	dirInfo, ok := res.entry.(*files.FolderMetadata)
 	if !ok {
 		return nil, fs.ErrorIsFile
 	}
@@ -836,16 +996,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	var res *files.ListFolderResult
 	for {
 		if !started {
-			arg := files.ListFolderArg{
-				Path:      f.opt.Enc.FromStandardPath(root),
-				Recursive: false,
-				Limit:     1000,
-			}
+			arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root))
+			arg.Recursive = false
+			arg.Limit = 1000
+
 			if root == "/" {
 				arg.Path = "" // Specify root folder as empty string
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolder(&arg)
+				res, err = f.srv.ListFolder(arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -898,7 +1057,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if err != nil {
 					return nil, err
 				}
-				entries = append(entries, o)
+				if o.(*Object).exportType.listable() {
+					entries = append(entries, o)
+				}
 			}
 		}
 		if !res.HasMore {
@@ -984,16 +1145,14 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		}
 
 		// check directory empty
-		arg := files.ListFolderArg{
-			Path:      encRoot,
-			Recursive: false,
-		}
+		arg := files.NewListFolderArg(encRoot)
+		arg.Recursive = false
 		if root == "/" {
 			arg.Path = "" // Specify root folder as empty string
 		}
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
-			res, err = f.srv.ListFolder(&arg)
+			res, err = f.srv.ListFolder(arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1174,6 +1333,16 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return shouldRetry(ctx, err)
 	})
 
+	if err != nil && createArg.Settings.Expires != nil && strings.Contains(err.Error(), sharing.SharedLinkSettingsErrorNotAuthorized) {
+		// Some plans can't create links with expiry
+		fs.Debugf(absPath, "can't create link with expiry, trying without")
+		createArg.Settings.Expires = nil
+		err = f.pacer.Call(func() (bool, error) {
+			linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+			return shouldRetry(ctx, err)
+		})
+	}
+
 	if err != nil && strings.Contains(err.Error(),
 		sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
 		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
@@ -1338,16 +1507,14 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 	var startCursor *files.ListFolderGetLatestCursorResult
 
 	err = f.pacer.Call(func() (bool, error) {
-		arg := files.ListFolderArg{
-			Path:      f.opt.Enc.FromStandardPath(f.slashRoot),
-			Recursive: true,
-		}
+		arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(f.slashRoot))
+		arg.Recursive = true
 
 		if arg.Path == "/" {
 			arg.Path = ""
 		}
 
-		startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
+		startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
 
 		return shouldRetry(ctx, err)
 	})
@@ -1451,8 +1618,50 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+func (f *Fs) chooseExportFormat(info *files.FileMetadata) (exportAPIFormat, exportExtension) {
+	// Find API export formats Dropbox supports for this file
+	// Sometimes Dropbox lists a format in ExportAs but not ExportOptions, so check both
+	ei := info.ExportInfo
+	dropboxFormatStrings := append([]string{ei.ExportAs}, ei.ExportOptions...)
+
+	// Find which extensions these correspond to
+	exportExtensions := map[exportExtension]exportAPIFormat{}
+	var dropboxPreferredAPIFormat exportAPIFormat
+	var dropboxPreferredExtension exportExtension
+	for _, format := range dropboxFormatStrings {
+		apiFormat := exportAPIFormat(format)
+		// Only consider formats we know about
+		if ext, ok := exportKnownAPIFormats[apiFormat]; ok {
+			if dropboxPreferredAPIFormat == "" {
+				dropboxPreferredAPIFormat = apiFormat
+				dropboxPreferredExtension = ext
+			}
+			exportExtensions[ext] = apiFormat
+		}
+	}
+
+	// See if the user picked a valid extension
+	for _, ext := range f.exportExts {
+		if apiFormat, ok := exportExtensions[ext]; ok {
+			return apiFormat, ext
+		}
+	}
+
+	// If no matches, prefer the first valid format Dropbox lists
+	return dropboxPreferredAPIFormat, dropboxPreferredExtension
+}
+
 // ------------------------------------------------------------
 
+func (et exportType) listable() bool {
+	return et != exportHide
+}
+
+// something we should _try_ to export
+func (et exportType) exportable() bool {
+	return et == exportExportable || et == exportListOnly
+}
+
 // Fs returns the parent Fs
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -1496,6 +1705,32 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
+func (o *Object) setMetadataForExport(info *files.FileMetadata) {
+	o.bytes = -1
+	o.hash = ""
+
+	if o.fs.opt.SkipExports {
+		o.exportType = exportHide
+		return
+	}
+	if o.fs.opt.ShowAllExports {
+		o.exportType = exportListOnly
+		return
+	}
+
+	var exportExt exportExtension
+	o.exportAPIFormat, exportExt = o.fs.chooseExportFormat(info)
+	if o.exportAPIFormat == "" {
+		o.exportType = exportHide
+	} else {
+		o.exportType = exportExportable
+		// get rid of any paper extension, if present
+		o.remote = strings.TrimSuffix(o.remote, paperExtension)
+		// add the export extension
+		o.remote += "." + string(exportExt)
+	}
+}
+
 // setMetadataFromEntry sets the fs data from a files.FileMetadata
 //
 // This isn't a complete set of metadata and has an inaccurate date
@@ -1504,6 +1739,10 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
 	o.bytes = int64(info.Size)
 	o.modTime = info.ClientModified
 	o.hash = info.ContentHash
+
+	if !info.IsDownloadable {
+		o.setMetadataForExport(info)
+	}
 	return nil
 }
 
@@ -1567,6 +1806,27 @@ func (o *Object) Storable() bool {
 	return true
 }
 
+func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
+	if o.exportType == exportListOnly || o.exportAPIFormat == "" {
+		fs.Debugf(o.remote, "No export format found")
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
+	var exportResult *files.ExportResult
+	err = o.fs.pacer.Call(func() (bool, error) {
+		exportResult, in, err = o.fs.srv.Export(&arg)
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	o.bytes = int64(exportResult.ExportMetadata.Size)
+	o.hash = exportResult.ExportMetadata.ExportHash
+	return
+}
+
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	if o.fs.opt.SharedFiles {
@@ -1586,6 +1846,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return
 	}
 
+	if o.exportType.exportable() {
+		return o.export(ctx)
+	}
+
 	fs.FixRangeOption(options, o.bytes)
 	headers := fs.OpenOptionHeaders(options)
 	arg := files.DownloadArg{

backend/dropbox/dropbox_internal_test.go

@@ -1,9 +1,16 @@
 package dropbox
 
 import (
+	"context"
+	"io"
+	"strings"
 	"testing"
 
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestInternalCheckPathLength(t *testing.T) {
@@ -42,3 +49,54 @@ func TestInternalCheckPathLength(t *testing.T) {
 		assert.Equal(t, test.ok, err == nil, test.in)
 	}
 }
+
+func (f *Fs) importPaperForTest(t *testing.T) {
+	content := `# test doc
+
+Lorem ipsum __dolor__ sit amet
+[link](http://google.com)
+`
+
+	arg := files.PaperCreateArg{
+		Path:         f.slashRootSlash + "export.paper",
+		ImportFormat: &files.ImportFormat{Tagged: dropbox.Tagged{Tag: files.ImportFormatMarkdown}},
+	}
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		reader := strings.NewReader(content)
+		_, err = f.srv.PaperCreate(&arg, reader)
+		return shouldRetry(context.Background(), err)
+	})
+	require.NoError(t, err)
+}
+
+func (f *Fs) InternalTestPaperExport(t *testing.T) {
+	ctx := context.Background()
+	f.importPaperForTest(t)
+
+	f.exportExts = []exportExtension{"html"}
+
+	obj, err := f.NewObject(ctx, "export.html")
+	require.NoError(t, err)
+
+	rc, err := obj.Open(ctx)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, rc.Close()) }()
+
+	buf, err := io.ReadAll(rc)
+	require.NoError(t, err)
+	text := string(buf)
+
+	for _, excerpt := range []string{
+		"Lorem ipsum",
+		"<b>dolor</b>",
+		`href="http://google.com"`,
+	} {
+		require.Contains(t, text, excerpt)
+	}
+}
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("PaperExport", f.InternalTestPaperExport)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/filefabric/api/types.go

@@ -216,11 +216,11 @@ var ItemFields = mustFields(Item{})
 
 // fields returns the JSON fields in use by opt as a | separated
 // string.
-func fields(opt interface{}) (pipeTags string, err error) {
+func fields(opt any) (pipeTags string, err error) {
 	var tags []string
 	def := reflect.ValueOf(opt)
 	defType := def.Type()
-	for i := 0; i < def.NumField(); i++ {
+	for i := range def.NumField() {
 		field := defType.Field(i)
 		tag, ok := field.Tag.Lookup("json")
 		if !ok {
@@ -239,7 +239,7 @@ func fields(opt interface{}) (pipeTags string, err error) {
 
 // mustFields returns the JSON fields in use by opt as a | separated
 // string. It panics on failure.
-func mustFields(opt interface{}) string {
+func mustFields(opt any) string {
 	tags, err := fields(opt)
 	if err != nil {
 		panic(err)
@@ -351,12 +351,12 @@ type SpaceInfo struct {
 // DeleteResponse is returned from doDeleteFile
 type DeleteResponse struct {
 	Status
-	Deleted        []string      `json:"deleted"`
-	Errors         []interface{} `json:"errors"`
-	ID             string        `json:"fi_id"`
-	BackgroundTask int           `json:"backgroundtask"`
-	UsSize         string        `json:"us_size"`
-	PaSize         string        `json:"pa_size"`
+	Deleted        []string `json:"deleted"`
+	Errors         []any    `json:"errors"`
+	ID             string   `json:"fi_id"`
+	BackgroundTask int      `json:"backgroundtask"`
+	UsSize         string   `json:"us_size"`
+	PaSize         string   `json:"pa_size"`
 	//SpaceInfo      SpaceInfo     `json:"spaceinfo"`
 }
 

backend/filefabric/filefabric.go

@@ -371,7 +371,7 @@ func (f *Fs) getToken(ctx context.Context) (token string, err error) {
 }
 
 // params for rpc
-type params map[string]interface{}
+type params map[string]any
 
 // rpc calls the rpc.php method of the SME file fabric
 //

backend/filescom/filescom.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 	"time"
 
@@ -169,11 +170,9 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 	}
 
 	if apiErr, ok := err.(files_sdk.ResponseError); ok {
-		for _, e := range retryErrorCodes {
-			if apiErr.HttpCode == e {
-				fs.Debugf(nil, "Retrying API error %v", err)
-				return true, err
-			}
+		if slices.Contains(retryErrorCodes, apiErr.HttpCode) {
+			fs.Debugf(nil, "Retrying API error %v", err)
+			return true, err
 		}
 	}
 

backend/ftp/ftp_internal_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type settings map[string]interface{}
+type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 	fsName := strings.Split(f.Name(), "{")[0] // strip off hash

backend/googlephotos/albums.go

@@ -4,6 +4,7 @@ package googlephotos
 
 import (
 	"path"
+	"slices"
 	"strings"
 	"sync"
 
@@ -119,7 +120,7 @@ func (as *albums) _del(album *api.Album) {
 		dirs := as.path[dir]
 		for i, dir := range dirs {
 			if dir == leaf {
-				dirs = append(dirs[:i], dirs[i+1:]...)
+				dirs = slices.Delete(dirs, i, i+1)
 				break
 			}
 		}

backend/googlephotos/googlephotos.go

@@ -388,7 +388,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
 		Method:  "GET",
 		RootURL: "https://accounts.google.com/.well-known/openid-configuration",
 	}
-	var openIDconfig map[string]interface{}
+	var openIDconfig map[string]any
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
 		return shouldRetry(ctx, resp, err)
@@ -448,7 +448,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 			"token_type_hint": []string{"access_token"},
 		},
 	}
-	var res interface{}
+	var res any
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
 		return shouldRetry(ctx, resp, err)

backend/googlephotos/googlephotos_test.go

@@ -2,6 +2,7 @@ package googlephotos
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -35,7 +36,7 @@ func TestIntegration(t *testing.T) {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
-	if err == fs.ErrorNotFoundInConfigFile {
+	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
 		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)

backend/hasher/commands.go

@@ -24,7 +24,7 @@ import (
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "drop":
 		return nil, f.db.Stop(true)

backend/hasher/kv.go

@@ -6,6 +6,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"maps"
 	"strings"
 	"time"
 
@@ -195,9 +196,7 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
 		r.Fp = op.fp
 	}
 
-	for hashType, hashVal := range op.hashes {
-		r.Hashes[hashType] = hashVal
-	}
+	maps.Copy(r.Hashes, op.hashes)
 	if data, err = r.encode(op.key); err != nil {
 		return fmt.Errorf("marshal failed: %w", err)
 	}

backend/hidrive/hidrivehash/hidrivehash.go

@@ -52,10 +52,7 @@ func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *ui
 	total := len(p)
 	nullBytes := make([]byte, blockSize)
 	for len(p) > 0 {
-		toWrite := int(blockSize - *bytesInBlock)
-		if toWrite > len(p) {
-			toWrite = len(p)
-		}
+		toWrite := min(int(blockSize-*bytesInBlock), len(p))
 		c, err := writer.Write(p[:toWrite])
 		*bytesInBlock += uint32(c)
 		*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
@@ -276,7 +273,7 @@ func (h *hidriveHash) Sum(b []byte) []byte {
 	}
 
 	checksum := zeroSum
-	for i := 0; i < len(h.levels); i++ {
+	for i := range h.levels {
 		level := h.levels[i]
 		if i < len(h.levels)-1 {
 			// Aggregate non-empty non-final levels.

backend/hidrive/hidrivehash/hidrivehash_test.go

@@ -216,7 +216,7 @@ func TestLevelWrite(t *testing.T) {
 func TestLevelIsFull(t *testing.T) {
 	content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
 	l := hidrivehash.NewLevel()
-	for i := 0; i < 256; i++ {
+	for range 256 {
 		assert.False(t, l.(internal.LevelHash).IsFull())
 		written, err := l.Write(content[:])
 		assert.Equal(t, len(content), written)

backend/http/http.go

@@ -505,7 +505,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		entries = append(entries, entry)
 		entriesMu.Unlock()
 	}
-	for i := 0; i < checkers; i++ {
+	for range checkers {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -740,7 +740,7 @@ It doesn't return anything.
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "set":
 		newOpt := f.opt

backend/iclouddrive/api/client.go

@@ -76,7 +76,7 @@ func (c *Client) DriveService() (*DriveService, error) {
 // This function is the main entry point for making requests to the iCloud
 // API. If the initial request returns a 401 (Unauthorized), it will try to
 // reauthenticate and retry the request.
-func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
 	resp, err = c.Session.Request(ctx, opts, request, response)
 	if err != nil && resp != nil {
 		// try to reauth
@@ -100,7 +100,7 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{
 // This function is useful when you have a session that is already
 // authenticated, but you need to make a request without triggering
 // a re-authentication.
-func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
 	// Make the request without re-authenticating
 	resp, err = c.Session.Request(ctx, opts, request, response)
 	return resp, err
@@ -161,6 +161,6 @@ func newRequestError(Status string, Text string) *RequestError {
 }
 
 // newErr orf makes a new error from sprintf parameters.
-func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError {
+func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {
 	return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
 }

backend/iclouddrive/api/drive.go

@@ -476,7 +476,7 @@ func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID st
 
 // CopyDocByItemID copies a document by its item ID.
 func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) {
-	// putting name in info doesnt work. extension does work so assume this is a bug in the endpoint
+	// putting name in info doesn't work. extension does work so assume this is a bug in the endpoint
 	values := map[string]any{
 		"info_to_update": map[string]any{},
 	}
@@ -733,8 +733,8 @@ type DocumentUpdateResponse struct {
 			StatusCode   int    `json:"status_code"`
 			ErrorMessage string `json:"error_message"`
 		} `json:"status"`
-		OperationID interface{} `json:"operation_id"`
-		Document    *Document   `json:"document"`
+		OperationID any       `json:"operation_id"`
+		Document    *Document `json:"document"`
 	} `json:"results"`
 }
 
@@ -765,9 +765,9 @@ type Document struct {
 		IsWritable   bool `json:"is_writable"`
 		IsHidden     bool `json:"is_hidden"`
 	} `json:"file_flags"`
-	LastOpenedTime   int64       `json:"lastOpenedTime"`
-	RestorePath      interface{} `json:"restorePath"`
-	HasChainedParent bool        `json:"hasChainedParent"`
+	LastOpenedTime   int64 `json:"lastOpenedTime"`
+	RestorePath      any   `json:"restorePath"`
+	HasChainedParent bool  `json:"hasChainedParent"`
 }
 
 // DriveID returns the drive ID of the Document.

backend/iclouddrive/api/session.go

@@ -3,13 +3,13 @@ package api
 import (
 	"context"
 	"fmt"
+	"maps"
 	"net/http"
 	"net/url"
 	"slices"
 	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
-
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -35,7 +35,7 @@ type Session struct {
 // }
 
 // Request makes a request
-func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) {
+func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {
 	resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
 
 	if err != nil {
@@ -129,7 +129,7 @@ func (s *Session) AuthWithToken(ctx context.Context) error {
 
 // Validate2FACode validates the 2FA code
 func (s *Session) Validate2FACode(ctx context.Context, code string) error {
-	values := map[string]interface{}{"securityCode": map[string]string{"code": code}}
+	values := map[string]any{"securityCode": map[string]string{"code": code}}
 	body, err := IntoReader(values)
 	if err != nil {
 		return err
@@ -220,9 +220,7 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
 		"Referer":                          fmt.Sprintf("%s/", homeEndpoint),
 		"User-Agent":                       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
 	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -230,9 +228,7 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
 func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
 	headers := GetCommonHeaders(map[string]string{})
 	headers["Cookie"] = s.GetCookieString()
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -254,9 +250,7 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
 		"Referer":      fmt.Sprintf("%s/", baseEndpoint),
 		"User-Agent":   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
 	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
+	maps.Copy(headers, overwrite)
 	return headers
 }
 
@@ -338,33 +332,33 @@ type AccountInfo struct {
 
 // ValidateDataDsInfo represents an validation info
 type ValidateDataDsInfo struct {
-	HsaVersion                         int           `json:"hsaVersion"`
-	LastName                           string        `json:"lastName"`
-	ICDPEnabled                        bool          `json:"iCDPEnabled"`
-	TantorMigrated                     bool          `json:"tantorMigrated"`
-	Dsid                               string        `json:"dsid"`
-	HsaEnabled                         bool          `json:"hsaEnabled"`
-	IsHideMyEmailSubscriptionActive    bool          `json:"isHideMyEmailSubscriptionActive"`
-	IroncadeMigrated                   bool          `json:"ironcadeMigrated"`
-	Locale                             string        `json:"locale"`
-	BrZoneConsolidated                 bool          `json:"brZoneConsolidated"`
-	ICDRSCapableDeviceList             string        `json:"ICDRSCapableDeviceList"`
-	IsManagedAppleID                   bool          `json:"isManagedAppleID"`
-	IsCustomDomainsFeatureAvailable    bool          `json:"isCustomDomainsFeatureAvailable"`
-	IsHideMyEmailFeatureAvailable      bool          `json:"isHideMyEmailFeatureAvailable"`
-	ContinueOnDeviceEligibleDeviceInfo []string      `json:"ContinueOnDeviceEligibleDeviceInfo"`
-	Gilligvited                        bool          `json:"gilligvited"`
-	AppleIDAliases                     []interface{} `json:"appleIdAliases"`
-	UbiquityEOLEnabled                 bool          `json:"ubiquityEOLEnabled"`
-	IsPaidDeveloper                    bool          `json:"isPaidDeveloper"`
-	CountryCode                        string        `json:"countryCode"`
-	NotificationID                     string        `json:"notificationId"`
-	PrimaryEmailVerified               bool          `json:"primaryEmailVerified"`
-	ADsID                              string        `json:"aDsID"`
-	Locked                             bool          `json:"locked"`
-	ICDRSCapableDeviceCount            int           `json:"ICDRSCapableDeviceCount"`
-	HasICloudQualifyingDevice          bool          `json:"hasICloudQualifyingDevice"`
-	PrimaryEmail                       string        `json:"primaryEmail"`
+	HsaVersion                         int      `json:"hsaVersion"`
+	LastName                           string   `json:"lastName"`
+	ICDPEnabled                        bool     `json:"iCDPEnabled"`
+	TantorMigrated                     bool     `json:"tantorMigrated"`
+	Dsid                               string   `json:"dsid"`
+	HsaEnabled                         bool     `json:"hsaEnabled"`
+	IsHideMyEmailSubscriptionActive    bool     `json:"isHideMyEmailSubscriptionActive"`
+	IroncadeMigrated                   bool     `json:"ironcadeMigrated"`
+	Locale                             string   `json:"locale"`
+	BrZoneConsolidated                 bool     `json:"brZoneConsolidated"`
+	ICDRSCapableDeviceList             string   `json:"ICDRSCapableDeviceList"`
+	IsManagedAppleID                   bool     `json:"isManagedAppleID"`
+	IsCustomDomainsFeatureAvailable    bool     `json:"isCustomDomainsFeatureAvailable"`
+	IsHideMyEmailFeatureAvailable      bool     `json:"isHideMyEmailFeatureAvailable"`
+	ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`
+	Gilligvited                        bool     `json:"gilligvited"`
+	AppleIDAliases                     []any    `json:"appleIdAliases"`
+	UbiquityEOLEnabled                 bool     `json:"ubiquityEOLEnabled"`
+	IsPaidDeveloper                    bool     `json:"isPaidDeveloper"`
+	CountryCode                        string   `json:"countryCode"`
+	NotificationID                     string   `json:"notificationId"`
+	PrimaryEmailVerified               bool     `json:"primaryEmailVerified"`
+	ADsID                              string   `json:"aDsID"`
+	Locked                             bool     `json:"locked"`
+	ICDRSCapableDeviceCount            int      `json:"ICDRSCapableDeviceCount"`
+	HasICloudQualifyingDevice          bool     `json:"hasICloudQualifyingDevice"`
+	PrimaryEmail                       string   `json:"primaryEmail"`
 	AppleIDEntries                     []struct {
 		IsPrimary bool   `json:"isPrimary"`
 		Type      string `json:"type"`

backend/iclouddrive/iclouddrive.go

@@ -445,7 +445,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// build request
-	// cant use normal rename as file needs to be "activated" first
+	// can't use normal rename as file needs to be "activated" first
 
 	r := api.NewUpdateFileInfo()
 	r.DocumentID = doc.DocumentID

backend/imagekit/client/media.go

@@ -75,7 +75,7 @@ type MoveFolderParam struct {
 	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
 }
 
-// JobIDResponse respresents response struct with JobID for folder operations
+// JobIDResponse represents response struct with JobID for folder operations
 type JobIDResponse struct {
 	JobID string `json:"jobId"`
 }

backend/imagekit/util.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"time"
 
@@ -142,12 +143,7 @@ func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
 	if resp == nil {
 		return false
 	}
-	for _, e := range retryErrorCodes {
-		if resp.StatusCode == e {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(retryErrorCodes, resp.StatusCode)
 }
 
 func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {

backend/internetarchive/internetarchive.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"path"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -151,6 +152,19 @@ Owner is able to add custom keys. Metadata feature grabs all the keys including
 			Help:     "Host of InternetArchive Frontend.\n\nLeave blank for default value.",
 			Default:  "https://archive.org",
 			Advanced: true,
+		}, {
+			Name: "item_metadata",
+			Help: `Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set.
+Format is key=value and the 'x-archive-meta-' prefix is automatically added.`,
+			Default:  []string{},
+			Hide:     fs.OptionHideConfigurator,
+			Advanced: true,
+		}, {
+			Name: "item_derive",
+			Help: `Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload.
+The derive process produces a number of secondary files from an upload to make an upload more usable on the web.
+Setting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure.`,
+			Default: true,
 		}, {
 			Name: "disable_checksum",
 			Help: `Don't ask the server to test against MD5 checksum calculated by rclone.
@@ -187,7 +201,7 @@ Only enable if you need to be guaranteed to be reflected after write operations.
 const iaItemMaxSize int64 = 1099511627776
 
 // metadata keys that are not writeable
-var roMetadataKey = map[string]interface{}{
+var roMetadataKey = map[string]any{
 	// do not add mtime here, it's a documented exception
 	"name": nil, "source": nil, "size": nil, "md5": nil,
 	"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
@@ -201,6 +215,8 @@ type Options struct {
 	Endpoint        string               `config:"endpoint"`
 	FrontEndpoint   string               `config:"front_endpoint"`
 	DisableChecksum bool                 `config:"disable_checksum"`
+	ItemMetadata    []string             `config:"item_metadata"`
+	ItemDerive      bool                 `config:"item_derive"`
 	WaitArchive     fs.Duration          `config:"wait_archive"`
 	Enc             encoder.MultiEncoder `config:"encoding"`
 }
@@ -790,17 +806,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
-		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
-		"x-archive-auto-make-bucket": "1",    // same as above in IAS3 original way
-		"x-archive-keep-old-version": "0",    // do not keep old versions (a.k.a. trashes in other clouds)
-		"x-archive-meta-mediatype":   "data", // mark media type of the uploading file as "data"
-		"x-archive-queue-derive":     "0",    // skip derivation process (e.g. encoding to smaller files, OCR on PDFs)
-		"x-archive-cascade-delete":   "1",    // enable "cascate delete" (delete all derived files in addition to the file itself)
+		"x-amz-auto-make-bucket":     "1", // create an item if does not exist, do nothing if already
+		"x-archive-auto-make-bucket": "1", // same as above in IAS3 original way
+		"x-archive-keep-old-version": "0", // do not keep old versions (a.k.a. trashes in other clouds)
+		"x-archive-cascade-delete":   "1", // enable "cascate delete" (delete all derived files in addition to the file itself)
 	}
+
 	if size >= 0 {
 		headers["Content-Length"] = fmt.Sprintf("%d", size)
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
+
+	// This is IA's ITEM metadata, not file metadata
+	headers, err = o.appendItemMetadataHeaders(headers, o.fs.opt)
+	if err != nil {
+		return err
+	}
+
 	var mdata fs.Metadata
 	mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
 	if err == nil && mdata != nil {
@@ -863,6 +885,51 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return err
 }
 
+func (o *Object) appendItemMetadataHeaders(headers map[string]string, options Options) (newHeaders map[string]string, err error) {
+	metadataCounter := make(map[string]int)
+	metadataValues := make(map[string][]string)
+
+	// First pass: count occurrences and collect values
+	for _, v := range options.ItemMetadata {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) != 2 {
+			return newHeaders, errors.New("item metadata key=value should be in the form key=value")
+		}
+		key, value := parts[0], parts[1]
+		metadataCounter[key]++
+		metadataValues[key] = append(metadataValues[key], value)
+	}
+
+	// Second pass: add headers with appropriate prefixes
+	for key, count := range metadataCounter {
+		if count == 1 {
+			// Only one occurrence, use x-archive-meta-
+			headers[fmt.Sprintf("x-archive-meta-%s", key)] = metadataValues[key][0]
+		} else {
+			// Multiple occurrences, use x-archive-meta01-, x-archive-meta02-, etc.
+			for i, value := range metadataValues[key] {
+				headers[fmt.Sprintf("x-archive-meta%02d-%s", i+1, key)] = value
+			}
+		}
+	}
+
+	if o.fs.opt.ItemDerive {
+		headers["x-archive-queue-derive"] = "1"
+	} else {
+		headers["x-archive-queue-derive"] = "0"
+	}
+
+	fs.Debugf(o, "Setting IA item derive: %t", o.fs.opt.ItemDerive)
+
+	for k, v := range headers {
+		if strings.HasPrefix(k, "x-archive-meta") {
+			fs.Debugf(o, "Setting IA item metadata: %s=%s", k, v)
+		}
+	}
+
+	return headers, nil
+}
+
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
@@ -925,10 +992,8 @@ func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error) {
 
 func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	if resp != nil {
-		for _, e := range retryErrorCodes {
-			if resp.StatusCode == e {
-				return true, err
-			}
+		if slices.Contains(retryErrorCodes, resp.StatusCode) {
+			return true, err
 		}
 	}
 	// Ok, not an awserr, check for generic failure conditions
@@ -1081,13 +1146,7 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSiz
 			}
 
 			fileTrackers, _ := listOrString(iaFile.UpdateTrack)
-			trackerMatch := false
-			for _, v := range fileTrackers {
-				if v == tracker {
-					trackerMatch = true
-					break
-				}
-			}
+			trackerMatch := slices.Contains(fileTrackers, tracker)
 			if !trackerMatch {
 				continue
 			}

backend/jottacloud/api/types.go

@@ -70,7 +70,7 @@ func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // MarshalJSON turns a Rfc3339Time into JSON
 func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
+	return fmt.Appendf(nil, "\"%s\"", t.String()), nil
 }
 
 // LoginToken is struct representing the login token generated in the WebUI
@@ -165,25 +165,25 @@ type DeviceRegistrationResponse struct {
 
 // CustomerInfo provides general information about the account. Required for finding the correct internal username.
 type CustomerInfo struct {
-	Username          string      `json:"username"`
-	Email             string      `json:"email"`
-	Name              string      `json:"name"`
-	CountryCode       string      `json:"country_code"`
-	LanguageCode      string      `json:"language_code"`
-	CustomerGroupCode string      `json:"customer_group_code"`
-	BrandCode         string      `json:"brand_code"`
-	AccountType       string      `json:"account_type"`
-	SubscriptionType  string      `json:"subscription_type"`
-	Usage             int64       `json:"usage"`
-	Quota             int64       `json:"quota"`
-	BusinessUsage     int64       `json:"business_usage"`
-	BusinessQuota     int64       `json:"business_quota"`
-	WriteLocked       bool        `json:"write_locked"`
-	ReadLocked        bool        `json:"read_locked"`
-	LockedCause       interface{} `json:"locked_cause"`
-	WebHash           string      `json:"web_hash"`
-	AndroidHash       string      `json:"android_hash"`
-	IOSHash           string      `json:"ios_hash"`
+	Username          string `json:"username"`
+	Email             string `json:"email"`
+	Name              string `json:"name"`
+	CountryCode       string `json:"country_code"`
+	LanguageCode      string `json:"language_code"`
+	CustomerGroupCode string `json:"customer_group_code"`
+	BrandCode         string `json:"brand_code"`
+	AccountType       string `json:"account_type"`
+	SubscriptionType  string `json:"subscription_type"`
+	Usage             int64  `json:"usage"`
+	Quota             int64  `json:"quota"`
+	BusinessUsage     int64  `json:"business_usage"`
+	BusinessQuota     int64  `json:"business_quota"`
+	WriteLocked       bool   `json:"write_locked"`
+	ReadLocked        bool   `json:"read_locked"`
+	LockedCause       any    `json:"locked_cause"`
+	WebHash           string `json:"web_hash"`
+	AndroidHash       string `json:"android_hash"`
+	IOSHash           string `json:"ios_hash"`
 }
 
 // TrashResponse is returned when emptying the Trash

backend/linkbox/linkbox.go

@@ -193,7 +193,7 @@ func (o *Object) set(e *entity) {
 // Call linkbox with the query in opts and return result
 //
 // This will be checked for error and an error will be returned if Status != 1
-func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result interface{}) error {
+func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result any) error {
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, opts, nil, &result)
 		return f.shouldRetry(ctx, resp, err)

backend/local/local.go

@@ -1046,7 +1046,7 @@ you can try to change the output.`,
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
 	switch name {
 	case "noop":
 		if txt, ok := opt["error"]; ok {
@@ -1056,7 +1056,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			return nil, errors.New(txt)
 		}
 		if _, ok := opt["echo"]; ok {
-			out := map[string]interface{}{}
+			out := map[string]any{}
 			out["name"] = name
 			out["arg"] = arg
 			out["opt"] = opt

backend/local/local_internal_test.go

@@ -86,7 +86,7 @@ func TestVerifyCopy(t *testing.T) {
 	require.NoError(t, err)
 	src.(*Object).fs.opt.NoCheckUpdated = true
 
-	for i := 0; i < 100; i++ {
+	for i := range 100 {
 		go r.WriteFile(src.Remote(), fmt.Sprintf("some new content %d", i), src.ModTime(context.Background()))
 	}
 	_, err = operations.Copy(context.Background(), r.Fremote, nil, filePath+"2", src)

backend/mailru/api/m1.go

@@ -63,8 +63,8 @@ type UserInfoResponse struct {
 				Prolong      bool   `json:"prolong"`
 				Promocodes   struct {
 				} `json:"promocodes"`
-				Subscription []interface{} `json:"subscription"`
-				Version      string        `json:"version"`
+				Subscription []any  `json:"subscription"`
+				Version      string `json:"version"`
 			} `json:"billing"`
 			Bonuses struct {
 				CameraUpload bool `json:"camera_upload"`

backend/mailru/mailru.go

@@ -901,7 +901,7 @@ func (t *treeState) NextRecord() (fs.DirEntry, error) {
 		return nil, nil
 	case api.ListParseUnknown15:
 		skip := int(r.ReadPu32())
-		for i := 0; i < skip; i++ {
+		for range skip {
 			r.ReadPu32()
 			r.ReadPu32()
 		}
@@ -1768,7 +1768,7 @@ func (f *Fs) eligibleForSpeedup(remote string, size int64, options ...fs.OpenOpt
 func (f *Fs) parseSpeedupPatterns(patternString string) (err error) {
 	f.speedupGlobs = nil
 	f.speedupAny = false
-	uniqueValidPatterns := make(map[string]interface{})
+	uniqueValidPatterns := make(map[string]any)
 
 	for _, pattern := range strings.Split(patternString, ",") {
 		pattern = strings.ToLower(strings.TrimSpace(pattern))
@@ -2131,10 +2131,7 @@ func getTransferRange(size int64, options ...fs.OpenOption) (start int64, end in
 	if limit < 0 {
 		limit = size - offset
 	}
-	end = offset + limit
-	if end > size {
-		end = size
-	}
+	end = min(offset+limit, size)
 	partial = !(offset == 0 && end == size)
 	return offset, end, partial
 }

backend/mailru/mrhash/mrhash_test.go

@@ -11,7 +11,7 @@ import (
 
 func testChunk(t *testing.T, chunk int) {
 	data := make([]byte, chunk)
-	for i := 0; i < chunk; i++ {
+	for i := range chunk {
 		data[i] = 'A'
 	}
 	for _, test := range []struct {

backend/mega/mega.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -218,11 +219,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
 		srv.SetHTTPS(opt.UseHTTPS)
-		srv.SetLogger(func(format string, v ...interface{}) {
+		srv.SetLogger(func(format string, v ...any) {
 			fs.Infof("*go-mega*", format, v...)
 		})
 		if opt.Debug {
-			srv.SetDebugger(func(format string, v ...interface{}) {
+			srv.SetDebugger(func(format string, v ...any) {
 				fs.Debugf("*go-mega*", format, v...)
 			})
 		}
@@ -498,11 +499,8 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
 	if err != nil {
 		return false, fmt.Errorf("list failed: %w", err)
 	}
-	for _, item := range nodes {
-		if fn(item) {
-			found = true
-			break
-		}
+	if slices.ContainsFunc(nodes, fn) {
+		found = true
 	}
 	return
 }
@@ -1156,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload the chunks
 	// FIXME do this in parallel
-	for id := 0; id < u.Chunks(); id++ {
+	for id := range u.Chunks() {
 		_, chunkSize, err := u.ChunkLocation(id)
 		if err != nil {
 			return fmt.Errorf("upload failed to read chunk location: %w", err)

backend/memory/memory_internal_test.go

@@ -29,7 +29,7 @@ func testPurgeListDeadlock(t *testing.T) {
 	r.Fremote.Features().Disable("Purge") // force fallback-purge
 
 	// make a lot of files to prevent it from finishing too quickly
-	for i := 0; i < 100; i++ {
+	for i := range 100 {
 		dst := "file" + fmt.Sprint(i) + ".txt"
 		r.WriteObject(ctx, dst, "hello", t1)
 	}

backend/netstorage/netstorage.go

@@ -274,7 +274,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 }
 
 // Command the backend to run a named commands: du and symlink
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "du":
 		// No arg parsing needed, the path is passed in the fs
@@ -858,7 +858,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 
 // callBackend calls NetStorage API using either rest.Call or rest.CallXML function,
 // depending on whether the response is required
-func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response interface{}, options []fs.OpenOption) (io.ReadCloser, error) {
+func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response any, options []fs.OpenOption) (io.ReadCloser, error) {
 	opts := rest.Opts{
 		Method:     method,
 		RootURL:    URL,
@@ -1080,7 +1080,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 }
 
 // netStorageDuRequest performs a NetStorage du request
-func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
+func (f *Fs) netStorageDuRequest(ctx context.Context) (any, error) {
 	URL := f.url("")
 	const actionHeader = "version=1&action=du&format=xml&encoding=utf-8"
 	duResp := &Du{}
@@ -1100,7 +1100,7 @@ func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
 }
 
 // netStorageDuRequest performs a NetStorage symlink request
-func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (interface{}, error) {
+func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (any, error) {
 	target := url.QueryEscape(strings.TrimSuffix(dst, "/"))
 	actionHeader := "version=1&action=symlink&target=" + target
 	if modTime != nil {

backend/onedrive/onedrive.go

@@ -131,7 +131,7 @@ func init() {
 					Help:  "Microsoft Cloud for US Government",
 				}, {
 					Value: regionDE,
-					Help:  "Microsoft Cloud Germany",
+					Help:  "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).",
 				}, {
 					Value: regionCN,
 					Help:  "Azure and Office 365 operated by Vnet Group in China",
@@ -2532,10 +2532,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 	remaining := size
 	position := int64(0)
 	for remaining > 0 {
-		n := int64(o.fs.opt.ChunkSize)
-		if remaining < n {
-			n = remaining
-		}
+		n := min(remaining, int64(o.fs.opt.ChunkSize))
 		seg := readers.NewRepeatableReader(io.LimitReader(in, n))
 		fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
 		info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)

backend/onedrive/quickxorhash/quickxorhash.go

@@ -86,7 +86,7 @@ func (q *quickXorHash) Write(p []byte) (n int, err error) {
 
 // Calculate the current checksum
 func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
-	for i := 0; i < dataSize; i++ {
+	for i := range dataSize {
 		shift := (i * 11) % 160
 		shiftBytes := shift / 8
 		shiftBits := shift % 8

backend/onedrive/quickxorhash/quickxorhash_test.go

@@ -130,10 +130,7 @@ func TestQuickXorHashByBlock(t *testing.T) {
 			require.NoError(t, err, what)
 			h := New()
 			for i := 0; i < len(in); i += blockSize {
-				end := i + blockSize
-				if end > len(in) {
-					end = len(in)
-				}
+				end := min(i+blockSize, len(in))
 				n, err := h.Write(in[i:end])
 				require.Equal(t, end-i, n, what)
 				require.NoError(t, err, what)

backend/opendrive/opendrive.go

@@ -92,6 +92,21 @@ Note that these chunks are buffered in memory so increasing them will
 increase memory use.`,
 			Default:  10 * fs.Mebi,
 			Advanced: true,
+		}, {
+			Name:     "access",
+			Help:     "Files and folders will be uploaded with this access permission (default private)",
+			Default:  "private",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "private",
+				Help:  "The file or folder access can be granted in a way that will allow select users to view, read or write what is absolutely essential for them.",
+			}, {
+				Value: "public",
+				Help:  "The file or folder can be downloaded by anyone from a web browser. The link can be shared in any way,",
+			}, {
+				Value: "hidden",
+				Help:  "The file or folder can be accessed has the same restrictions as  Public if the user knows the URL of the file or folder link in order to access the contents",
+			}},
 		}},
 	})
 }
@@ -102,6 +117,7 @@ type Options struct {
 	Password  string               `config:"password"`
 	Enc       encoder.MultiEncoder `config:"encoding"`
 	ChunkSize fs.SizeSuffix        `config:"chunk_size"`
+	Access    string               `config:"access"`
 }
 
 // Fs represents a remote server
@@ -475,7 +491,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Method: "POST",
 		Path:   "/file/move_copy.json",
 	}
-	var request interface{} = moveCopyFileData
+	var request any = moveCopyFileData
 
 	// use /file/rename.json if moving within the same directory
 	_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
@@ -548,7 +564,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		Method: "POST",
 		Path:   "/folder/move_copy.json",
 	}
-	var request interface{} = moveFolderData
+	var request any = moveFolderData
 
 	// use /folder/rename.json if moving within the same parent directory
 	if srcDirectoryID == dstDirectoryID {
@@ -735,6 +751,23 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
+// getAccessLevel is a helper function to determine access level integer
+func getAccessLevel(access string) int64 {
+	var accessLevel int64
+	switch access {
+	case "private":
+		accessLevel = 0
+	case "public":
+		accessLevel = 1
+	case "hidden":
+		accessLevel = 2
+	default:
+		accessLevel = 0
+		fs.Errorf(nil, "Invalid access: %s, defaulting to private", access)
+	}
+	return accessLevel
+}
+
 // DirCacher methods
 
 // CreateDir makes a directory with pathID as parent and name leaf
@@ -747,7 +780,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			SessionID:           f.session.SessionID,
 			FolderName:          f.opt.Enc.FromStandardName(leaf),
 			FolderSubParent:     pathID,
-			FolderIsPublic:      0,
+			FolderIsPublic:      getAccessLevel(f.opt.Access),
 			FolderPublicUpl:     0,
 			FolderPublicDisplay: 0,
 			FolderPublicDnl:     0,
@@ -1009,10 +1042,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	chunkCounter := 0
 
 	for remainingBytes > 0 {
-		currentChunkSize := int64(o.fs.opt.ChunkSize)
-		if currentChunkSize > remainingBytes {
-			currentChunkSize = remainingBytes
-		}
+		currentChunkSize := min(int64(o.fs.opt.ChunkSize), remainingBytes)
 		remainingBytes -= currentChunkSize
 		fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
 
@@ -1080,7 +1110,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Set permissions
 	err = o.fs.pacer.Call(func() (bool, error) {
-		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: 0}
+		update := permissions{SessionID: o.fs.session.SessionID, FileID: o.id, FileIsPublic: getAccessLevel(o.fs.opt.Access)}
 		// fs.Debugf(nil, "Permissions : %#v", update)
 		opts := rest.Opts{
 			Method:     "POST",

backend/oracleobjectstorage/command.go

@@ -131,7 +131,7 @@ If it is a string or a []string it will be shown to the user
 otherwise it will be JSON encoded and shown to the user like that
 */
 func (f *Fs) Command(ctx context.Context, commandName string, args []string,
-	opt map[string]string) (result interface{}, err error) {
+	opt map[string]string) (result any, err error) {
 	// fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt)
 	switch commandName {
 	case operationRename:
@@ -159,7 +159,7 @@ func (f *Fs) Command(ctx context.Context, commandName string, args []string,
 	}
 }
 
-func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) {
+func (f *Fs) rename(ctx context.Context, remote, newName string) (any, error) {
 	if remote == "" {
 		return nil, fmt.Errorf("path to object file cannot be empty")
 	}
@@ -332,7 +332,7 @@ func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPat
 	return uploadedParts, nil
 }
 
-func (f *Fs) restore(ctx context.Context, opt map[string]string) (interface{}, error) {
+func (f *Fs) restore(ctx context.Context, opt map[string]string) (any, error) {
 	req := objectstorage.RestoreObjectsRequest{
 		NamespaceName:         common.String(f.opt.Namespace),
 		RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{},

backend/oracleobjectstorage/copy.go

@@ -112,7 +112,7 @@ func copyObjectWaitForWorkRequest(ctx context.Context, wID *string, entityType s
 			string(objectstorage.WorkRequestSummaryStatusCanceled),
 			string(objectstorage.WorkRequestStatusFailed),
 		},
-		Refresh: func() (interface{}, string, error) {
+		Refresh: func() (any, string, error) {
 			getWorkRequestRequest := objectstorage.GetWorkRequestRequest{}
 			getWorkRequestRequest.WorkRequestId = wID
 			workRequestResponse, err := client.GetWorkRequest(context.Background(), getWorkRequestRequest)

backend/oracleobjectstorage/object.go

@@ -131,7 +131,7 @@ func (o *Object) setMetaData(
 	contentMd5 *string,
 	contentType *string,
 	lastModified *common.SDKTime,
-	storageTier interface{},
+	storageTier any,
 	meta map[string]string) error {
 
 	if contentLength != nil {

backend/oracleobjectstorage/waiter.go

@@ -5,6 +5,7 @@ package oracleobjectstorage
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -23,7 +24,7 @@ var refreshGracePeriod = 30 * time.Second
 //
 // `state` is the latest state of that object. And `err` is any error that
 // may have happened while refreshing the state.
-type StateRefreshFunc func() (result interface{}, state string, err error)
+type StateRefreshFunc func() (result any, state string, err error)
 
 // StateChangeConf is the configuration struct used for `WaitForState`.
 type StateChangeConf struct {
@@ -56,7 +57,7 @@ type StateChangeConf struct {
 // reach the target state.
 //
 // Cancellation from the passed in context will cancel the refresh loop
-func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (interface{}, error) {
+func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (any, error) {
 	// fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target)
 
 	notfoundTick := 0
@@ -72,7 +73,7 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 	}
 
 	type Result struct {
-		Result interface{}
+		Result any
 		State  string
 		Error  error
 		Done   bool
@@ -165,12 +166,9 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 					}
 				}
 
-				for _, allowed := range conf.Pending {
-					if currentState == allowed {
-						found = true
-						targetOccurrence = 0
-						break
-					}
+				if slices.Contains(conf.Pending, currentState) {
+					found = true
+					targetOccurrence = 0
 				}
 
 				if !found && len(conf.Pending) > 0 {
@@ -278,8 +276,8 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
 // NotFoundError resource not found error
 type NotFoundError struct {
 	LastError    error
-	LastRequest  interface{}
-	LastResponse interface{}
+	LastRequest  any
+	LastResponse any
 	Message      string
 	Retries      int
 }

backend/pcloud/pcloud.go

@@ -424,7 +424,7 @@ func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) {
 	})
 	// Set our own http client in the context
 	ctx = oauthutil.Context(ctx, baseClient)
-	// create a new oauth client, re-use the token source
+	// create a new oauth client, reuse the token source
 	oAuthClient := oauth2.NewClient(ctx, f.ts)
 	return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil
 }
@@ -990,10 +990,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	if err != nil {
 		return nil, err
 	}
-	free := q.Quota - q.UsedQuota
-	if free < 0 {
-		free = 0
-	}
+	free := max(q.Quota-q.UsedQuota, 0)
 	usage = &fs.Usage{
 		Total: fs.NewUsageValue(q.Quota),     // quota of bytes that can be used
 		Used:  fs.NewUsageValue(q.UsedQuota), // bytes in use
@@ -1324,7 +1321,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		// sometimes pcloud leaves a half complete file on
 		// error, so delete it if it exists, trying a few times
-		for i := 0; i < 5; i++ {
+		for range 5 {
 			delObj, delErr := o.fs.NewObject(ctx, o.remote)
 			if delErr == nil && delObj != nil {
 				_ = delObj.Remove(ctx)

backend/pcloud/writer_at.go

@@ -37,7 +37,7 @@ func (c *writerAt) Close() error {
 	}
 	sizeOk := false
 	sizeLastSeen := int64(0)
-	for retry := 0; retry < 5; retry++ {
+	for retry := range 5 {
 		fs.Debugf(c.remote, "checking file size: try %d/5", retry)
 		obj, err := c.fs.NewObject(c.ctx, c.remote)
 		if err != nil {

backend/pikpak/api/types.go

@@ -71,14 +71,14 @@ type Error struct {
 
 // ErrorDetails contains further details of api error
 type ErrorDetails struct {
-	Type         string        `json:"@type,omitempty"`
-	Reason       string        `json:"reason,omitempty"`
-	Domain       string        `json:"domain,omitempty"`
-	Metadata     struct{}      `json:"metadata,omitempty"` // TODO: undiscovered yet
-	Locale       string        `json:"locale,omitempty"`   // e.g. "en"
-	Message      string        `json:"message,omitempty"`
-	StackEntries []interface{} `json:"stack_entries,omitempty"` // TODO: undiscovered yet
-	Detail       string        `json:"detail,omitempty"`
+	Type         string   `json:"@type,omitempty"`
+	Reason       string   `json:"reason,omitempty"`
+	Domain       string   `json:"domain,omitempty"`
+	Metadata     struct{} `json:"metadata,omitempty"` // TODO: undiscovered yet
+	Locale       string   `json:"locale,omitempty"`   // e.g. "en"
+	Message      string   `json:"message,omitempty"`
+	StackEntries []any    `json:"stack_entries,omitempty"` // TODO: undiscovered yet
+	Detail       string   `json:"detail,omitempty"`
 }
 
 // Error returns a string for the error and satisfies the error interface
@@ -168,44 +168,44 @@ type FileList struct {
 // for a single file, i.e. supports for higher `--multi-thread-streams=N`.
 // However, it is not generally applicable as it is only for media.
 type File struct {
-	Apps              []*FileApp    `json:"apps,omitempty"`
-	Audit             *FileAudit    `json:"audit,omitempty"`
-	Collection        string        `json:"collection,omitempty"` // TODO
-	CreatedTime       Time          `json:"created_time,omitempty"`
-	DeleteTime        Time          `json:"delete_time,omitempty"`
-	FileCategory      string        `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
-	FileExtension     string        `json:"file_extension,omitempty"`
-	FolderType        string        `json:"folder_type,omitempty"`
-	Hash              string        `json:"hash,omitempty"` // custom hash with a form of sha1sum
-	IconLink          string        `json:"icon_link,omitempty"`
-	ID                string        `json:"id,omitempty"`
-	Kind              string        `json:"kind,omitempty"` // "drive#file"
-	Links             *FileLinks    `json:"links,omitempty"`
-	Md5Checksum       string        `json:"md5_checksum,omitempty"`
-	Medias            []*Media      `json:"medias,omitempty"`
-	MimeType          string        `json:"mime_type,omitempty"`
-	ModifiedTime      Time          `json:"modified_time,omitempty"` // updated when renamed or moved
-	Name              string        `json:"name,omitempty"`
-	OriginalFileIndex int           `json:"original_file_index,omitempty"` // TODO
-	OriginalURL       string        `json:"original_url,omitempty"`
-	Params            *FileParams   `json:"params,omitempty"`
-	ParentID          string        `json:"parent_id,omitempty"`
-	Phase             string        `json:"phase,omitempty"`
-	Revision          int           `json:"revision,omitempty,string"`
-	ReferenceEvents   []interface{} `json:"reference_events"`
-	ReferenceResource interface{}   `json:"reference_resource"`
-	Size              int64         `json:"size,omitempty,string"`
-	SortName          string        `json:"sort_name,omitempty"`
-	Space             string        `json:"space,omitempty"`
-	SpellName         []interface{} `json:"spell_name,omitempty"` // TODO maybe list of something?
-	Starred           bool          `json:"starred,omitempty"`
-	Tags              []interface{} `json:"tags"`
-	ThumbnailLink     string        `json:"thumbnail_link,omitempty"`
-	Trashed           bool          `json:"trashed,omitempty"`
-	UserID            string        `json:"user_id,omitempty"`
-	UserModifiedTime  Time          `json:"user_modified_time,omitempty"`
-	WebContentLink    string        `json:"web_content_link,omitempty"`
-	Writable          bool          `json:"writable,omitempty"`
+	Apps              []*FileApp  `json:"apps,omitempty"`
+	Audit             *FileAudit  `json:"audit,omitempty"`
+	Collection        string      `json:"collection,omitempty"` // TODO
+	CreatedTime       Time        `json:"created_time,omitempty"`
+	DeleteTime        Time        `json:"delete_time,omitempty"`
+	FileCategory      string      `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
+	FileExtension     string      `json:"file_extension,omitempty"`
+	FolderType        string      `json:"folder_type,omitempty"`
+	Hash              string      `json:"hash,omitempty"` // custom hash with a form of sha1sum
+	IconLink          string      `json:"icon_link,omitempty"`
+	ID                string      `json:"id,omitempty"`
+	Kind              string      `json:"kind,omitempty"` // "drive#file"
+	Links             *FileLinks  `json:"links,omitempty"`
+	Md5Checksum       string      `json:"md5_checksum,omitempty"`
+	Medias            []*Media    `json:"medias,omitempty"`
+	MimeType          string      `json:"mime_type,omitempty"`
+	ModifiedTime      Time        `json:"modified_time,omitempty"` // updated when renamed or moved
+	Name              string      `json:"name,omitempty"`
+	OriginalFileIndex int         `json:"original_file_index,omitempty"` // TODO
+	OriginalURL       string      `json:"original_url,omitempty"`
+	Params            *FileParams `json:"params,omitempty"`
+	ParentID          string      `json:"parent_id,omitempty"`
+	Phase             string      `json:"phase,omitempty"`
+	Revision          int         `json:"revision,omitempty,string"`
+	ReferenceEvents   []any       `json:"reference_events"`
+	ReferenceResource any         `json:"reference_resource"`
+	Size              int64       `json:"size,omitempty,string"`
+	SortName          string      `json:"sort_name,omitempty"`
+	Space             string      `json:"space,omitempty"`
+	SpellName         []any       `json:"spell_name,omitempty"` // TODO maybe list of something?
+	Starred           bool        `json:"starred,omitempty"`
+	Tags              []any       `json:"tags"`
+	ThumbnailLink     string      `json:"thumbnail_link,omitempty"`
+	Trashed           bool        `json:"trashed,omitempty"`
+	UserID            string      `json:"user_id,omitempty"`
+	UserModifiedTime  Time        `json:"user_modified_time,omitempty"`
+	WebContentLink    string      `json:"web_content_link,omitempty"`
+	Writable          bool        `json:"writable,omitempty"`
 }
 
 // FileLinks includes links to file at backend
@@ -235,18 +235,18 @@ type Media struct {
 		VideoType  string `json:"video_type,omitempty"`  // "mpegts"
 		HdrType    string `json:"hdr_type,omitempty"`
 	} `json:"video,omitempty"`
-	Link           *Link         `json:"link,omitempty"`
-	NeedMoreQuota  bool          `json:"need_more_quota,omitempty"`
-	VipTypes       []interface{} `json:"vip_types,omitempty"` // TODO maybe list of something?
-	RedirectLink   string        `json:"redirect_link,omitempty"`
-	IconLink       string        `json:"icon_link,omitempty"`
-	IsDefault      bool          `json:"is_default,omitempty"`
-	Priority       int           `json:"priority,omitempty"`
-	IsOrigin       bool          `json:"is_origin,omitempty"`
-	ResolutionName string        `json:"resolution_name,omitempty"`
-	IsVisible      bool          `json:"is_visible,omitempty"`
-	Category       string        `json:"category,omitempty"` // "category_origin"
-	Audio          interface{}   `json:"audio"`              // TODO: undiscovered yet
+	Link           *Link  `json:"link,omitempty"`
+	NeedMoreQuota  bool   `json:"need_more_quota,omitempty"`
+	VipTypes       []any  `json:"vip_types,omitempty"` // TODO maybe list of something?
+	RedirectLink   string `json:"redirect_link,omitempty"`
+	IconLink       string `json:"icon_link,omitempty"`
+	IsDefault      bool   `json:"is_default,omitempty"`
+	Priority       int    `json:"priority,omitempty"`
+	IsOrigin       bool   `json:"is_origin,omitempty"`
+	ResolutionName string `json:"resolution_name,omitempty"`
+	IsVisible      bool   `json:"is_visible,omitempty"`
+	Category       string `json:"category,omitempty"` // "category_origin"
+	Audio          any    `json:"audio"`              // TODO: undiscovered yet
 }
 
 // FileParams includes parameters for instant open
@@ -263,20 +263,20 @@ type FileParams struct {
 
 // FileApp includes parameters for instant open
 type FileApp struct {
-	ID            string        `json:"id,omitempty"`   // "decompress" for rar files
-	Name          string        `json:"name,omitempty"` // decompress" for rar files
-	Access        []interface{} `json:"access,omitempty"`
-	Link          string        `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
-	RedirectLink  string        `json:"redirect_link,omitempty"`
-	VipTypes      []interface{} `json:"vip_types,omitempty"`
-	NeedMoreQuota bool          `json:"need_more_quota,omitempty"`
-	IconLink      string        `json:"icon_link,omitempty"`
-	IsDefault     bool          `json:"is_default,omitempty"`
-	Params        struct{}      `json:"params,omitempty"` // TODO
-	CategoryIDs   []interface{} `json:"category_ids,omitempty"`
-	AdSceneType   int           `json:"ad_scene_type,omitempty"`
-	Space         string        `json:"space,omitempty"`
-	Links         struct{}      `json:"links,omitempty"` // TODO
+	ID            string   `json:"id,omitempty"`   // "decompress" for rar files
+	Name          string   `json:"name,omitempty"` // decompress" for rar files
+	Access        []any    `json:"access,omitempty"`
+	Link          string   `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
+	RedirectLink  string   `json:"redirect_link,omitempty"`
+	VipTypes      []any    `json:"vip_types,omitempty"`
+	NeedMoreQuota bool     `json:"need_more_quota,omitempty"`
+	IconLink      string   `json:"icon_link,omitempty"`
+	IsDefault     bool     `json:"is_default,omitempty"`
+	Params        struct{} `json:"params,omitempty"` // TODO
+	CategoryIDs   []any    `json:"category_ids,omitempty"`
+	AdSceneType   int      `json:"ad_scene_type,omitempty"`
+	Space         string   `json:"space,omitempty"`
+	Links         struct{} `json:"links,omitempty"` // TODO
 }
 
 // ------------------------------------------------------------
@@ -290,27 +290,27 @@ type TaskList struct {
 
 // Task is a basic element representing a single task such as offline download and upload
 type Task struct {
-	Kind              string        `json:"kind,omitempty"` // "drive#task"
-	ID                string        `json:"id,omitempty"`   // task id?
-	Name              string        `json:"name,omitempty"` // torrent name?
-	Type              string        `json:"type,omitempty"` // "offline"
-	UserID            string        `json:"user_id,omitempty"`
-	Statuses          []interface{} `json:"statuses,omitempty"`    // TODO
-	StatusSize        int           `json:"status_size,omitempty"` // TODO
-	Params            *TaskParams   `json:"params,omitempty"`      // TODO
-	FileID            string        `json:"file_id,omitempty"`
-	FileName          string        `json:"file_name,omitempty"`
-	FileSize          string        `json:"file_size,omitempty"`
-	Message           string        `json:"message,omitempty"` // e.g. "Saving"
-	CreatedTime       Time          `json:"created_time,omitempty"`
-	UpdatedTime       Time          `json:"updated_time,omitempty"`
-	ThirdTaskID       string        `json:"third_task_id,omitempty"` // TODO
-	Phase             string        `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
-	Progress          int           `json:"progress,omitempty"`
-	IconLink          string        `json:"icon_link,omitempty"`
-	Callback          string        `json:"callback,omitempty"`
-	ReferenceResource interface{}   `json:"reference_resource,omitempty"` // TODO
-	Space             string        `json:"space,omitempty"`
+	Kind              string      `json:"kind,omitempty"` // "drive#task"
+	ID                string      `json:"id,omitempty"`   // task id?
+	Name              string      `json:"name,omitempty"` // torrent name?
+	Type              string      `json:"type,omitempty"` // "offline"
+	UserID            string      `json:"user_id,omitempty"`
+	Statuses          []any       `json:"statuses,omitempty"`    // TODO
+	StatusSize        int         `json:"status_size,omitempty"` // TODO
+	Params            *TaskParams `json:"params,omitempty"`      // TODO
+	FileID            string      `json:"file_id,omitempty"`
+	FileName          string      `json:"file_name,omitempty"`
+	FileSize          string      `json:"file_size,omitempty"`
+	Message           string      `json:"message,omitempty"` // e.g. "Saving"
+	CreatedTime       Time        `json:"created_time,omitempty"`
+	UpdatedTime       Time        `json:"updated_time,omitempty"`
+	ThirdTaskID       string      `json:"third_task_id,omitempty"` // TODO
+	Phase             string      `json:"phase,omitempty"`         // e.g. "PHASE_TYPE_RUNNING"
+	Progress          int         `json:"progress,omitempty"`
+	IconLink          string      `json:"icon_link,omitempty"`
+	Callback          string      `json:"callback,omitempty"`
+	ReferenceResource any         `json:"reference_resource,omitempty"` // TODO
+	Space             string      `json:"space,omitempty"`
 }
 
 // TaskParams includes parameters informing status of Task

backend/pikpak/helper.go

@@ -638,7 +638,7 @@ func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper
 	return c
 }
 
-func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request any, response any) (resp *http.Response, err error) {
 	if c.captcha != nil {
 		token, err := c.captcha.Token(opts)
 		if err != nil || token == "" {

backend/pikpak/pikpak.go

@@ -1232,7 +1232,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	params := url.Values{}
 	iVal := reflect.ValueOf(&form.MultiParts).Elem()
 	iTyp := iVal.Type()
-	for i := 0; i < iVal.NumField(); i++ {
+	for i := range iVal.NumField() {
 		params.Set(iTyp.Field(i).Tag.Get("json"), iVal.Field(i).String())
 	}
 	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, params, "file", name)
@@ -1520,7 +1520,7 @@ Result:
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "addurl":
 		if len(arg) != 1 {

backend/putio/error.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"time"
 
@@ -13,10 +14,8 @@ import (
 )
 
 func checkStatusCode(resp *http.Response, expected ...int) error {
-	for _, code := range expected {
-		if resp.StatusCode == code {
-			return nil
-		}
+	if slices.Contains(expected, resp.StatusCode) {
+		return nil
 	}
 	return &statusCodeError{response: resp}
 }

backend/putio/fs.go

@@ -332,10 +332,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
 	var offsetMismatch bool
 	buf := make([]byte, defaultChunkSize)
 	for clientOffset < size {
-		chunkSize := size - clientOffset
-		if chunkSize >= int64(defaultChunkSize) {
-			chunkSize = int64(defaultChunkSize)
-		}
+		chunkSize := min(size-clientOffset, int64(defaultChunkSize))
 		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
 		chunkStart := clientOffset
 		reqSize := chunkSize

backend/qingstor/upload.go

@@ -358,7 +358,7 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
 	})()
 
 	ch := make(chan chunk, mu.cfg.concurrency)
-	for i := 0; i < mu.cfg.concurrency; i++ {
+	for range mu.cfg.concurrency {
 		mu.wg.Add(1)
 		go mu.readChunk(ch)
 	}

backend/quatrix/quatrix.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -643,10 +644,8 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
 		return err
 	}
 
-	for _, removedID := range result.IDs {
-		if removedID == id {
-			return nil
-		}
+	if slices.Contains(result.IDs, id) {
+		return nil
 	}
 
 	return fmt.Errorf("file %s was not deleted successfully", id)

backend/quatrix/upload_memory.go

@@ -59,11 +59,7 @@ func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed f
 
 	defer func() { u.fileUsage[fileID] = borrowed }()
 
-	effectiveChunkSize := int64(speed * u.effectiveTime.Seconds())
-
-	if effectiveChunkSize < u.reserved {
-		effectiveChunkSize = u.reserved
-	}
+	effectiveChunkSize := max(int64(speed*u.effectiveTime.Seconds()), u.reserved)
 
 	if neededMemory < effectiveChunkSize {
 		effectiveChunkSize = neededMemory

backend/s3/ibm_signer.go

@@ -0,0 +1,59 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+)
+
+// Authenticator defines an interface for obtaining an IAM token.
+type Authenticator interface {
+	GetToken() (string, error)
+}
+
+// IbmIamSigner is a structure for signing requests using IBM IAM.
+// Requeres APIKey and Resource InstanceID
+type IbmIamSigner struct {
+	APIKey     string
+	InstanceID string
+	Auth       Authenticator
+}
+
+// SignHTTP signs requests using IBM IAM token.
+func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error {
+	var authenticator Authenticator
+	if signer.Auth != nil {
+		authenticator = signer.Auth
+	} else {
+		authenticator = &core.IamAuthenticator{ApiKey: signer.APIKey}
+	}
+	token, err := authenticator.GetToken()
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("ibm-service-instance-id", signer.InstanceID)
+	return nil
+}
+
+// NoOpCredentialsProvider is needed since S3 SDK requires having credentials, even though authentication is happening via IBM IAM.
+type NoOpCredentialsProvider struct{}
+
+// Retrieve returns mock credentials for the NoOpCredentialsProvider.
+func (n *NoOpCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID:     "NoOpAccessKey",
+		SecretAccessKey: "NoOpSecretKey",
+		SessionToken:    "",
+		Source:          "NoOpCredentialsProvider",
+	}, nil
+}
+
+// IsExpired always returns false
+func (n *NoOpCredentialsProvider) IsExpired() bool {
+	return false
+}

backend/s3/ibm_signer_test.go

@@ -0,0 +1,47 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/stretchr/testify/assert"
+)
+
+type MockAuthenticator struct {
+	Token string
+	Error error
+}
+
+func (m *MockAuthenticator) GetToken() (string, error) {
+	return m.Token, m.Error
+}
+
+func TestSignHTTP(t *testing.T) {
+	apiKey := "mock-api-key"
+	instanceID := "mock-instance-id"
+	token := "mock-iam-token"
+	mockAuth := &MockAuthenticator{
+		Token: token,
+		Error: nil,
+	}
+	signer := &IbmIamSigner{
+		APIKey:     apiKey,
+		InstanceID: instanceID,
+		Auth:       mockAuth,
+	}
+	req, err := http.NewRequest("GET", "https://example.com", nil)
+	if err != nil {
+		t.Fatalf("Failed to create HTTP request: %v", err)
+	}
+	credentials := aws.Credentials{
+		AccessKeyID:     "mock-access-key",
+		SecretAccessKey: "mock-secret-key",
+	}
+	err = signer.SignHTTP(context.TODO(), credentials, req, "payload-hash", "service", "region", time.Now())
+	assert.NoError(t, err, "Expected no error")
+	assert.Equal(t, "Bearer "+token, req.Header.Get("Authorization"), "Authorization header should be set correctly")
+	assert.Equal(t, instanceID, req.Header.Get("ibm-service-instance-id"), "ibm-service-instance-id header should be set correctly")
+}

backend/s3/s3.go

@@ -19,6 +19,7 @@ import (
 	"net/url"
 	"path"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -36,8 +37,8 @@ import (
 	"github.com/aws/smithy-go/logging"
 	"github.com/aws/smithy-go/middleware"
 	smithyhttp "github.com/aws/smithy-go/transport/http"
-
 	"github.com/ncw/swift/v2"
+
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -934,34 +935,67 @@ func init() {
 				Help:  "The default endpoint\nIran",
 			}},
 		}, {
-			// Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint
+			// Linode endpoints: https://techdocs.akamai.com/cloud-computing/docs/object-storage-product-limits#supported-endpoint-types-by-region
 			Name:     "endpoint",
 			Help:     "Endpoint for Linode Object Storage API.",
 			Provider: "Linode",
 			Examples: []fs.OptionExample{{
+				Value: "nl-ams-1.linodeobjects.com",
+				Help:  "Amsterdam (Netherlands), nl-ams-1",
+			}, {
 				Value: "us-southeast-1.linodeobjects.com",
 				Help:  "Atlanta, GA (USA), us-southeast-1",
+			}, {
+				Value: "in-maa-1.linodeobjects.com",
+				Help:  "Chennai (India), in-maa-1",
 			}, {
 				Value: "us-ord-1.linodeobjects.com",
 				Help:  "Chicago, IL (USA), us-ord-1",
 			}, {
 				Value: "eu-central-1.linodeobjects.com",
 				Help:  "Frankfurt (Germany), eu-central-1",
+			}, {
+				Value: "id-cgk-1.linodeobjects.com",
+				Help:  "Jakarta (Indonesia), id-cgk-1",
+			}, {
+				Value: "gb-lon-1.linodeobjects.com",
+				Help:  "London 2 (Great Britain), gb-lon-1",
+			}, {
+				Value: "us-lax-1.linodeobjects.com",
+				Help:  "Los Angeles, CA (USA), us-lax-1",
+			}, {
+				Value: "es-mad-1.linodeobjects.com",
+				Help:  "Madrid (Spain), es-mad-1",
+			}, {
+				Value: "au-mel-1.linodeobjects.com",
+				Help:  "Melbourne (Australia), au-mel-1",
+			}, {
+				Value: "us-mia-1.linodeobjects.com",
+				Help:  "Miami, FL (USA), us-mia-1",
 			}, {
 				Value: "it-mil-1.linodeobjects.com",
 				Help:  "Milan (Italy), it-mil-1",
 			}, {
 				Value: "us-east-1.linodeobjects.com",
 				Help:  "Newark, NJ (USA), us-east-1",
+			}, {
+				Value: "jp-osa-1.linodeobjects.com",
+				Help:  "Osaka (Japan), jp-osa-1",
 			}, {
 				Value: "fr-par-1.linodeobjects.com",
 				Help:  "Paris (France), fr-par-1",
+			}, {
+				Value: "br-gru-1.linodeobjects.com",
+				Help:  "São Paulo (Brazil), br-gru-1",
 			}, {
 				Value: "us-sea-1.linodeobjects.com",
 				Help:  "Seattle, WA (USA), us-sea-1",
 			}, {
 				Value: "ap-south-1.linodeobjects.com",
-				Help:  "Singapore ap-south-1",
+				Help:  "Singapore, ap-south-1",
+			}, {
+				Value: "sg-sin-1.linodeobjects.com",
+				Help:  "Singapore 2, sg-sin-1",
 			}, {
 				Value: "se-sto-1.linodeobjects.com",
 				Help:  "Stockholm (Sweden), se-sto-1",
@@ -1343,7 +1377,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1356,6 +1390,10 @@ func init() {
 				Value:    "sfo3.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces San Francisco 3",
 				Provider: "DigitalOcean",
+			}, {
+				Value:    "sfo2.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces San Francisco 2",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "fra1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Frankfurt 1",
@@ -1372,6 +1410,18 @@ func init() {
 				Value:    "sgp1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
+			}, {
+				Value:    "lon1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces London 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "tor1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Toronto 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "blr1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Bangalore 1",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
@@ -1476,14 +1526,6 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
-			}, {
-				Value:    "br-se1.magaluobjects.com",
-				Help:     "Magalu BR Southeast 1 endpoint",
-				Provider: "Magalu",
-			}, {
-				Value:    "br-ne1.magaluobjects.com",
-				Help:     "Magalu BR Northeast 1 endpoint",
-				Provider: "Magalu",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -2122,13 +2164,16 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "Standard storage class",
 			}},
 		}, {
-			// Mapping from here: #todo
+			// Mapping from here: https://docs.magalu.cloud/docs/storage/object-storage/Classes-de-Armazenamento/standard
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in Magalu.",
 			Provider: "Magalu",
 			Examples: []fs.OptionExample{{
 				Value: "STANDARD",
 				Help:  "Standard storage class",
+			}, {
+				Value: "GLACIER_IR",
+				Help:  "Glacier Instant Retrieval storage class",
 			}},
 		}, {
 			// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
@@ -2669,6 +2714,34 @@ knows about - please make a bug report if not.
 You can change this if you want to disable the use of multipart uploads.
 This shouldn't be necessary in normal operation.
 
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "use_x_id",
+			Help: `Set if rclone should add x-id URL parameters.
+
+You can change this if you want to disable the AWS SDK from
+adding x-id URL parameters.
+
+This shouldn't be necessary in normal operation.
+
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+			Default:  fs.Tristate{},
+			Advanced: true,
+		}, {
+			Name: "sign_accept_encoding",
+			Help: `Set if rclone should include Accept-Encoding as part of the signature.
+
+You can change this if you want to stop rclone including
+Accept-Encoding as part of the signature.
+
+This shouldn't be necessary in normal operation.
+
 This should be automatically set correctly for all providers rclone
 knows about - please make a bug report if not.
 `,
@@ -2725,6 +2798,16 @@ use |-vv| to see the debug level logs.
 			Default:  sdkLogMode(0),
 			Advanced: true,
 		},
+			{
+				Name:     "ibm_api_key",
+				Help:     "IBM API Key to be used to obtain IAM token",
+				Provider: "IBMCOS",
+			},
+			{
+				Name:     "ibm_resource_instance_id",
+				Help:     "IBM service instance id",
+				Provider: "IBMCOS",
+			},
 		}})
 }
 
@@ -2878,6 +2961,10 @@ type Options struct {
 	UseUnsignedPayload    fs.Tristate          `config:"use_unsigned_payload"`
 	SDKLogMode            sdkLogMode           `config:"sdk_log_mode"`
 	DirectoryBucket       bool                 `config:"directory_bucket"`
+	IBMAPIKey             string               `config:"ibm_api_key"`
+	IBMInstanceID         string               `config:"ibm_resource_instance_id"`
+	UseXID                fs.Tristate          `config:"use_x_id"`
+	SignAcceptEncoding    fs.Tristate          `config:"sign_accept_encoding"`
 }
 
 // Fs represents a remote s3 server
@@ -3011,10 +3098,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 				return true, err
 			}
 		}
-		for _, e := range retryErrorCodes {
-			if httpStatusCode == e {
-				return true, err
-			}
+		if slices.Contains(retryErrorCodes, httpStatusCode) {
+			return true, err
 		}
 	}
 	// Ok, not an awserr, check for generic failure conditions
@@ -3062,59 +3147,67 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 	}
 }
 
+// Fixup the request if needed.
+//
 // Google Cloud Storage alters the Accept-Encoding header, which
-// breaks the v2 request signature
+// breaks the v2 request signature. This is set with opt.SignAcceptEncoding.
 //
 // It also doesn't like the x-id URL parameter SDKv2 puts in so we
-// remove that too.
+// remove that too. This is set with opt.UseXID.Value.
 //
 // See https://github.com/aws/aws-sdk-go-v2/issues/1816.
 // Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
-func fixupGCS(o *s3.Options) {
+func fixupRequest(o *s3.Options, opt *Options) {
 	type ignoredHeadersKey struct{}
 	headers := []string{"Accept-Encoding"}
 
 	fixup := middleware.FinalizeMiddlewareFunc(
-		"FixupGCS",
+		"FixupRequest",
 		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
 			req, ok := in.Request.(*smithyhttp.Request)
 			if !ok {
-				return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+				return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
 			}
 
-			// Delete headers from being signed - will restore later
-			ignored := make(map[string]string, len(headers))
-			for _, h := range headers {
-				ignored[h] = req.Header.Get(h)
-				req.Header.Del(h)
+			if !opt.SignAcceptEncoding.Value {
+				// Delete headers from being signed - will restore later
+				ignored := make(map[string]string, len(headers))
+				for _, h := range headers {
+					ignored[h] = req.Header.Get(h)
+					req.Header.Del(h)
+				}
+
+				// Store ignored on context
+				ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
 			}
 
-			// Remove x-id because Google doesn't like them
-			if query := req.URL.Query(); query.Has("x-id") {
-				query.Del("x-id")
-				req.URL.RawQuery = query.Encode()
+			if !opt.UseXID.Value {
+				// Remove x-id
+				if query := req.URL.Query(); query.Has("x-id") {
+					query.Del("x-id")
+					req.URL.RawQuery = query.Encode()
+				}
 			}
 
-			// Store ignored on context
-			ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
-
 			return next.HandleFinalize(ctx, in)
 		},
 	)
 
 	// Restore headers if necessary
 	restore := middleware.FinalizeMiddlewareFunc(
-		"FixupGCSRestoreHeaders",
+		"FixupRequestRestoreHeaders",
 		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
 			req, ok := in.Request.(*smithyhttp.Request)
 			if !ok {
-				return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+				return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
 			}
 
-			// Restore ignored from ctx
-			ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
-			for k, v := range ignored {
-				req.Header.Set(k, v)
+			if !opt.SignAcceptEncoding.Value {
+				// Restore ignored from ctx
+				ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
+				for k, v := range ignored {
+					req.Header.Set(k, v)
+				}
 			}
 
 			return next.HandleFinalize(ctx, in)
@@ -3136,7 +3229,7 @@ func fixupGCS(o *s3.Options) {
 type s3logger struct{}
 
 // Logf is expected to support the standard fmt package "verbs".
-func (s3logger) Logf(classification logging.Classification, format string, v ...interface{}) {
+func (s3logger) Logf(classification logging.Classification, format string, v ...any) {
 	switch classification {
 	default:
 	case logging.Debug:
@@ -3160,6 +3253,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 
 	// Try to fill in the config from the environment if env_auth=true
 	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
+
 		configOpts := []func(*awsconfig.LoadOptions) error{}
 		// Set the name of the profile if supplied
 		if opt.Profile != "" {
@@ -3173,8 +3267,12 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 		if err != nil {
 			return nil, fmt.Errorf("couldn't load configuration with env_auth=true: %w", err)
 		}
+
 	} else {
 		switch {
+		case opt.Provider == "IBMCOS" && opt.V2Auth:
+			awsConfig.Credentials = &NoOpCredentialsProvider{}
+			fs.Debugf(nil, "Using IBM IAM")
 		case opt.AccessKeyID == "" && opt.SecretAccessKey == "":
 			// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
 			awsConfig.Credentials = aws.AnonymousCredentials{}
@@ -3228,14 +3326,21 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
 
 	if opt.V2Auth || opt.Region == "other-v2-signature" {
 		fs.Debugf(nil, "Using v2 auth")
-		options = append(options, func(s3Opt *s3.Options) {
-			s3Opt.HTTPSignerV4 = &v2Signer{opt: opt}
-		})
+		if opt.Provider == "IBMCOS" && opt.IBMAPIKey != "" && opt.IBMInstanceID != "" {
+			options = append(options, func(s3Opt *s3.Options) {
+				s3Opt.HTTPSignerV4 = &IbmIamSigner{APIKey: opt.IBMAPIKey, InstanceID: opt.IBMInstanceID}
+			})
+		} else {
+			options = append(options, func(s3Opt *s3.Options) {
+				s3Opt.HTTPSignerV4 = &v2Signer{opt: opt}
+			})
+		}
 	}
 
-	if opt.Provider == "GCS" {
+	// Fixup the request if needed
+	if !opt.UseXID.Value || !opt.SignAcceptEncoding.Value {
 		options = append(options, func(o *s3.Options) {
-			fixupGCS(o)
+			fixupRequest(o, opt)
 		})
 	}
 
@@ -3344,12 +3449,14 @@ func setQuirks(opt *Options) {
 		listObjectsV2         = true // Always use ListObjectsV2 instead of ListObjects
 		virtualHostStyle      = true // Use bucket.provider.com instead of putting the bucket in the URL
 		urlEncodeListings     = true // URL encode the listings to help with control characters
-		useMultipartEtag      = true // Set if Etags for multpart uploads are compatible with AWS
+		useMultipartEtag      = true // Set if Etags for multipart uploads are compatible with AWS
 		useAcceptEncodingGzip = true // Set Accept-Encoding: gzip
 		mightGzip             = true // assume all providers might use content encoding gzip until proven otherwise
 		useAlreadyExists      = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
 		useMultipartUploads   = true // Set if provider supports multipart uploads
 		useUnsignedPayload    = true // Do we need to use unsigned payloads to avoid seeking in PutObject
+		useXID                = true // Add x-id URL parameter into requests
+		signAcceptEncoding    = true // If we should include AcceptEncoding in the signature
 	)
 	switch opt.Provider {
 	case "AWS":
@@ -3494,11 +3601,14 @@ func setQuirks(opt *Options) {
 		// Google break request Signature by mutating accept-encoding HTTP header
 		// https://github.com/rclone/rclone/issues/6670
 		useAcceptEncodingGzip = false
+		signAcceptEncoding = false
 		useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
 		// GCS S3 doesn't support multi-part server side copy:
 		// See: https://issuetracker.google.com/issues/323465186
 		// So make cutoff very large which it does seem to support
 		opt.CopyCutoff = math.MaxInt64
+		// GCS doesn't like the x-id URL parameter the SDKv2 inserts
+		useXID = false
 	default: //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last case
 		fs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)
 		fallthrough
@@ -3568,6 +3678,18 @@ func setQuirks(opt *Options) {
 		opt.UseUnsignedPayload.Valid = true
 		opt.UseUnsignedPayload.Value = useUnsignedPayload
 	}
+
+	// Set the correct use UseXID if not manually set
+	if !opt.UseXID.Valid {
+		opt.UseXID.Valid = true
+		opt.UseXID.Value = useXID
+	}
+
+	// Set the correct SignAcceptEncoding if not manually set
+	if !opt.SignAcceptEncoding.Valid {
+		opt.SignAcceptEncoding.Valid = true
+		opt.SignAcceptEncoding.Value = signAcceptEncoding
+	}
 }
 
 // setRoot changes the root of the Fs
@@ -3682,6 +3804,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.Provider == "IDrive" {
 		f.features.SetTier = false
 	}
+	if opt.Provider == "AWS" {
+		f.features.DoubleSlash = true
+	}
 	if opt.DirectoryMarkers {
 		f.features.CanHaveEmptyDirectories = true
 	}
@@ -4153,7 +4278,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 		opt.prefix += "/"
 	}
 	if !opt.findFile {
-		if opt.directory != "" {
+		if opt.directory != "" && (opt.prefix == "" && !bucket.IsAllSlashes(opt.directory) || opt.prefix != "" && !strings.HasSuffix(opt.directory, "/")) {
 			opt.directory += "/"
 		}
 	}
@@ -4250,14 +4375,18 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 				}
 				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, opt.prefix) {
-					fs.Logf(f, "Odd name received %q", remote)
+					fs.Logf(f, "Odd directory name received %q", remote)
 					continue
 				}
 				remote = remote[len(opt.prefix):]
+				// Trim one slash off the remote name
+				remote, _ = strings.CutSuffix(remote, "/")
+				if remote == "" || bucket.IsAllSlashes(remote) {
+					remote += "/"
+				}
 				if opt.addBucket {
 					remote = bucket.Join(opt.bucket, remote)
 				}
-				remote = strings.TrimSuffix(remote, "/")
 				err = fn(remote, &types.Object{Key: &remote}, nil, true)
 				if err != nil {
 					if err == errEndList {
@@ -5123,7 +5252,7 @@ It doesn't return anything.
 // The result should be capable of being JSON encoded
 // If it is a string or a []string it will be shown to the user
 // otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
 	case "restore":
 		req := s3.RestoreObjectInput{
@@ -6057,7 +6186,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 			if mOut == nil {
 				err = fserrors.RetryErrorf("internal error: no info from multipart upload")
 			} else if mOut.UploadId == nil {
-				err = fserrors.RetryErrorf("internal error: no UploadId in multpart upload: %#v", *mOut)
+				err = fserrors.RetryErrorf("internal error: no UploadId in multipart upload: %#v", *mOut)
 			}
 		}
 		return f.shouldRetry(ctx, err)

backend/seafile/renew.go

@@ -9,9 +9,9 @@ import (
 
 // Renew allows tokens to be renewed on expiry.
 type Renew struct {
-	ts       *time.Ticker     // timer indicating when it's time to renew the token
-	run      func() error     // the callback to do the renewal
-	done     chan interface{} // channel to end the go routine
+	ts       *time.Ticker // timer indicating when it's time to renew the token
+	run      func() error // the callback to do the renewal
+	done     chan any     // channel to end the go routine
 	shutdown *sync.Once
 }
 
@@ -22,7 +22,7 @@ func NewRenew(every time.Duration, run func() error) *Renew {
 	r := &Renew{
 		ts:       time.NewTicker(every),
 		run:      run,
-		done:     make(chan interface{}),
+		done:     make(chan any),
 		shutdown: &sync.Once{},
 	}
 	go r.renewOnExpiry()

backend/seafile/seafile.go

@@ -1313,7 +1313,7 @@ func (f *Fs) getCachedLibraries(ctx context.Context) ([]api.Library, error) {
 	f.librariesMutex.Lock()
 	defer f.librariesMutex.Unlock()
 
-	libraries, err := f.libraries.Get(librariesCacheKey, func(key string) (value interface{}, ok bool, error error) {
+	libraries, err := f.libraries.Get(librariesCacheKey, func(key string) (value any, ok bool, error error) {
 		// Load the libraries if not present in the cache
 		libraries, err := f.getLibraries(ctx)
 		if err != nil {

backend/sftp/ssh_external.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"slices"
 	"strings"
 	"time"
 
@@ -89,7 +90,7 @@ func (f *Fs) newSSHSessionExternal() *sshSessionExternal {
 	// Connect to a remote host and request the sftp subsystem via
 	// the 'ssh' command. This assumes that passwordless login is
 	// correctly configured.
-	ssh := append([]string(nil), s.f.opt.SSH...)
+	ssh := slices.Clone(s.f.opt.SSH)
 	s.cmd = exec.CommandContext(ctx, ssh[0], ssh[1:]...)
 
 	// Allow the command a short time only to shut down

backend/sftp/stringlock_test.go

@@ -20,13 +20,13 @@ func TestStringLock(t *testing.T) {
 		inner = 100
 		total = outer * inner
 	)
-	for k := 0; k < outer; k++ {
+	for range outer {
 		for j := range counter {
 			wg.Add(1)
 			go func(j int) {
 				defer wg.Done()
 				ID := fmt.Sprintf("%d", j)
-				for i := 0; i < inner; i++ {
+				for range inner {
 					lock.Lock(ID)
 					n := counter[j]
 					time.Sleep(1 * time.Millisecond)

backend/sharefile/sharefile.go

@@ -537,7 +537,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 // Fill up (or reset) the buffer tokens
 func (f *Fs) fillBufferTokens() {
 	f.bufferTokens = make(chan []byte, f.ci.Transfers)
-	for i := 0; i < f.ci.Transfers; i++ {
+	for range f.ci.Transfers {
 		f.bufferTokens <- nil
 	}
 }

backend/sharefile/upload.go

@@ -57,10 +57,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		return nil, fmt.Errorf("can't use method %q with newLargeUpload", info.Method)
 	}
 
-	threads := f.ci.Transfers
-	if threads > info.MaxNumberOfThreads {
-		threads = info.MaxNumberOfThreads
-	}
+	threads := min(f.ci.Transfers, info.MaxNumberOfThreads)
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering

backend/sia/sia.go

@@ -337,7 +337,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	}
 
 	// Cleanup stray files left after failed upload
-	for i := 0; i < 5; i++ {
+	for range 5 {
 		cleanObj, cleanErr := f.NewObject(ctx, src.Remote())
 		if cleanErr == nil {
 			cleanErr = cleanObj.Remove(ctx)

backend/smb/connpool.go

@@ -2,8 +2,10 @@ package smb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"os"
 	"time"
 
 	smb2 "github.com/cloudsoda/go-smb2"
@@ -11,14 +13,17 @@ import (
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fshttp"
+	"golang.org/x/sync/errgroup"
 )
 
 // dial starts a client connection to the given SMB server. It is a
 // convenience function that connects to the given network address,
 // initiates the SMB handshake, and then sets up a Client.
+//
+// The context is only used for establishing the connection, not after.
 func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 	dialer := fshttp.NewDialer(ctx)
-	tconn, err := dialer.Dial(network, addr)
+	tconn, err := dialer.DialContext(ctx, network, addr)
 	if err != nil {
 		return nil, err
 	}
@@ -31,13 +36,29 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 		}
 	}
 
-	d := &smb2.Dialer{
-		Initiator: &smb2.NTLMInitiator{
+	d := &smb2.Dialer{}
+	if f.opt.UseKerberos {
+		cl, err := getKerberosClient()
+		if err != nil {
+			return nil, err
+		}
+
+		spn := f.opt.SPN
+		if spn == "" {
+			spn = "cifs/" + f.opt.Host
+		}
+
+		d.Initiator = &smb2.Krb5Initiator{
+			Client:    cl,
+			TargetSPN: spn,
+		}
+	} else {
+		d.Initiator = &smb2.NTLMInitiator{
 			User:      f.opt.User,
 			Password:  pass,
 			Domain:    f.opt.Domain,
 			TargetSPN: f.opt.SPN,
-		},
+		}
 	}
 
 	session, err := d.DialConn(ctx, tconn, addr)
@@ -73,15 +94,7 @@ func (c *conn) close() (err error) {
 
 // True if it's closed
 func (c *conn) closed() bool {
-	var nopErr error
-	if c.smbShare != nil {
-		// stat the current directory
-		_, nopErr = c.smbShare.Stat(".")
-	} else {
-		// list the shares
-		_, nopErr = c.smbSession.ListSharenames()
-	}
-	return nopErr != nil
+	return c.smbSession.Echo() != nil
 }
 
 // Show that we are using a SMB session
@@ -102,23 +115,20 @@ func (f *Fs) getSessions() int32 {
 }
 
 // Open a new connection to the SMB server.
+//
+// The context is only used for establishing the connection, not after.
 func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) {
-	// As we are pooling these connections we need to decouple
-	// them from the current context
-	bgCtx := context.Background()
-
-	c, err = f.dial(bgCtx, "tcp", f.opt.Host+":"+f.opt.Port)
+	c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SMB: %w", err)
 	}
 	if share != "" {
 		// mount the specified share as well if user requested
-		c.smbShare, err = c.smbSession.Mount(share)
+		err = c.mountShare(share)
 		if err != nil {
 			_ = c.smbSession.Logoff()
 			return nil, fmt.Errorf("couldn't initialize SMB: %w", err)
 		}
-		c.smbShare = c.smbShare.WithContext(bgCtx)
 	}
 	return c, nil
 }
@@ -176,23 +186,30 @@ func (f *Fs) getConnection(ctx context.Context, share string) (c *conn, err erro
 // Return a SMB connection to the pool
 //
 // It nils the pointed to connection out so it can't be reused
-func (f *Fs) putConnection(pc **conn) {
-	c := *pc
-	*pc = nil
-
-	var nopErr error
-	if c.smbShare != nil {
-		// stat the current directory
-		_, nopErr = c.smbShare.Stat(".")
-	} else {
-		// list the shares
-		_, nopErr = c.smbSession.ListSharenames()
-	}
-	if nopErr != nil {
-		fs.Debugf(f, "Connection failed, closing: %v", nopErr)
-		_ = c.close()
+//
+// if err is not nil then it checks the connection is alive using an
+// ECHO request
+func (f *Fs) putConnection(pc **conn, err error) {
+	if pc == nil {
 		return
 	}
+	c := *pc
+	if c == nil {
+		return
+	}
+	*pc = nil
+	if err != nil {
+		// If not a regular SMB error then check the connection
+		if !(errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrExist) || errors.Is(err, os.ErrPermission)) {
+			echoErr := c.smbSession.Echo()
+			if echoErr != nil {
+				fs.Debugf(f, "Connection failed, closing: %v", echoErr)
+				_ = c.close()
+				return
+			}
+			fs.Debugf(f, "Connection OK after error: %v", err)
+		}
+	}
 
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
@@ -219,15 +236,18 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
 	if len(f.pool) != 0 {
 		fs.Debugf(f, "Closing %d unused connections", len(f.pool))
 	}
+
+	g, _ := errgroup.WithContext(ctx)
 	for i, c := range f.pool {
-		if !c.closed() {
-			cErr := c.close()
-			if cErr != nil {
-				err = cErr
+		g.Go(func() (err error) {
+			if !c.closed() {
+				err = c.close()
 			}
-		}
-		f.pool[i] = nil
+			f.pool[i] = nil
+			return err
+		})
 	}
+	err = g.Wait()
 	f.pool = nil
 	return err
 }

backend/smb/kerberos.go

@@ -0,0 +1,78 @@
+package smb
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/jcmturner/gokrb5/v8/client"
+	"github.com/jcmturner/gokrb5/v8/config"
+	"github.com/jcmturner/gokrb5/v8/credentials"
+)
+
+var (
+	kerberosClient *client.Client
+	kerberosErr    error
+	kerberosOnce   sync.Once
+)
+
+// getKerberosClient returns a Kerberos client that can be used to authenticate.
+func getKerberosClient() (*client.Client, error) {
+	if kerberosClient == nil || kerberosErr == nil {
+		kerberosOnce.Do(func() {
+			kerberosClient, kerberosErr = createKerberosClient()
+		})
+	}
+
+	return kerberosClient, kerberosErr
+}
+
+// createKerberosClient creates a new Kerberos client.
+func createKerberosClient() (*client.Client, error) {
+	cfgPath := os.Getenv("KRB5_CONFIG")
+	if cfgPath == "" {
+		cfgPath = "/etc/krb5.conf"
+	}
+
+	cfg, err := config.Load(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine the ccache location from the environment, falling back to the
+	// default location.
+	ccachePath := os.Getenv("KRB5CCNAME")
+	switch {
+	case strings.Contains(ccachePath, ":"):
+		parts := strings.SplitN(ccachePath, ":", 2)
+		switch parts[0] {
+		case "FILE":
+			ccachePath = parts[1]
+		case "DIR":
+			primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
+			if err != nil {
+				return nil, err
+			}
+			ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
+		default:
+			return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
+		}
+	case ccachePath == "":
+		u, err := user.Current()
+		if err != nil {
+			return nil, err
+		}
+
+		ccachePath = "/tmp/krb5cc_" + u.Uid
+	}
+
+	ccache, err := credentials.LoadCCache(ccachePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.NewFromCCache(ccache, cfg)
+}

backend/smb/smb.go

@@ -25,7 +25,7 @@ import (
 )
 
 const (
-	minSleep      = 100 * time.Millisecond
+	minSleep      = 10 * time.Millisecond
 	maxSleep      = 2 * time.Second
 	decayConstant = 2 // bigger for slower decay, exponential
 )
@@ -76,6 +76,16 @@ authentication, and it often needs to be set for clusters. For example:
 Leave blank if not sure.
 `,
 			Sensitive: true,
+		}, {
+			Name: "use_kerberos",
+			Help: `Use Kerberos authentication.
+
+If set, rclone will use Kerberos authentication instead of NTLM. This
+requires a valid Kerberos configuration and credentials cache to be
+available, either in the default locations or as specified by the
+KRB5_CONFIG and KRB5CCNAME environment variables.
+`,
+			Default: false,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -126,6 +136,7 @@ type Options struct {
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
 	SPN             string      `config:"spn"`
+	UseKerberos     bool        `config:"use_kerberos"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`
@@ -196,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 	stat, err := cn.smbShare.Stat(f.toSambaPath(dir))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		// ignore stat error here
 		return f, nil
@@ -257,7 +268,7 @@ func (f *Fs) findObjectSeparate(ctx context.Context, share, path string) (fs.Obj
 		return nil, err
 	}
 	stat, err := cn.smbShare.Stat(f.toSambaPath(path))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, translateError(err, false)
 	}
@@ -279,7 +290,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 		return err
 	}
 	err = cn.smbShare.MkdirAll(f.toSambaPath(path), 0o755)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -294,7 +305,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return err
 	}
 	err = cn.smbShare.Remove(f.toSambaPath(path))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -364,7 +375,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		return nil, err
 	}
 	err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath))
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, translateError(err, false)
 	}
@@ -401,7 +412,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	if err != nil {
 		return err
 	}
-	defer f.putConnection(&cn)
+	defer f.putConnection(&cn, err)
 
 	_, err = cn.smbShare.Stat(dstPath)
 	if os.IsNotExist(err) {
@@ -419,7 +430,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if err != nil {
 		return nil, err
 	}
-	defer f.putConnection(&cn)
+	defer f.putConnection(&cn, err)
 
 	if share == "" {
 		shares, err := cn.smbSession.ListSharenames()
@@ -463,7 +474,7 @@ func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) {
 		return nil, err
 	}
 	stat, err := cn.smbShare.Statfs(dir)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	if err != nil {
 		return nil, err
 	}
@@ -545,7 +556,7 @@ func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error {
 		return err
 	}
 	err = cn.smbShare.MkdirAll(f.toSambaPath(dir), 0o755)
-	f.putConnection(&cn)
+	f.putConnection(&cn, err)
 	return err
 }
 
@@ -593,7 +604,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	if err != nil {
 		return err
 	}
-	defer o.fs.putConnection(&cn)
+	defer o.fs.putConnection(&cn, err)
 
 	err = cn.smbShare.Chtimes(reqDir, t, t)
 	if err != nil {
@@ -639,24 +650,25 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	fl, err := cn.smbShare.OpenFile(filename, os.O_RDONLY, 0)
 	if err != nil {
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 		return nil, fmt.Errorf("failed to open: %w", err)
 	}
 	pos, err := fl.Seek(offset, io.SeekStart)
 	if err != nil {
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 		return nil, fmt.Errorf("failed to seek: %w", err)
 	}
 	if pos != offset {
-		o.fs.putConnection(&cn)
-		return nil, fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos)
+		err = fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos)
+		o.fs.putConnection(&cn, err)
+		return nil, err
 	}
 
 	in = readers.NewLimitedReadCloser(fl, limit)
 	in = &boundReadCloser{
 		rc: in,
 		close: func() error {
-			o.fs.putConnection(&cn)
+			o.fs.putConnection(&cn, nil)
 			return nil
 		},
 	}
@@ -686,7 +698,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 	defer func() {
-		o.fs.putConnection(&cn)
+		o.fs.putConnection(&cn, err)
 	}()
 
 	fl, err := cn.smbShare.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
@@ -746,7 +758,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	}
 
 	err = cn.smbShare.Remove(filename)
-	o.fs.putConnection(&cn)
+	o.fs.putConnection(&cn, err)
 
 	return err
 }

backend/smb/smb_test.go

@@ -2,6 +2,7 @@
 package smb_test
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/rclone/rclone/backend/smb"
@@ -15,3 +16,13 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*smb.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	krb5Dir := t.TempDir()
+	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
+	t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSMBKerberos:rclone",
+		NilObject:  (*smb.Object)(nil),
+	})
+}