onedrive: add support for OAuth client credential flow - fixes #6197

This adds support for the client credential flow oauth method which
requires some special handling in onedrive:

- Special scopes are required
- The tenant is required
- The tenant needs to be used in the oauth URLs

This also:

- refactors the oauth config creation so it isn't duplicated
- defaults the drive_id to the previous one in the config
- updates the documentation

Co-authored-by: Nick Craig-Wood <nick@craig-wood.com>

.github/workflows/build.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
 
         include:
           - job_name: linux
@@ -80,6 +80,18 @@ jobs:
             compile_all: true
             deploy: true
 
+          - job_name: go1.21
+            os: ubuntu-latest
+            go: '1.21'
+            quicktest: true
+            racequicktest: true
+
+          - job_name: go1.22
+            os: ubuntu-latest
+            go: '1.22'
+            quicktest: true
+            racequicktest: true
+
     name: ${{ matrix.job_name }}
 
     runs-on: ${{ matrix.os }}

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,77 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .
+            - name: Build and publish image
+              uses: docker/build-push-action@v6
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha, scope=${{ github.workflow }}
+                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -1,294 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*-
-
-name: Build & Push Docker Images
-
-# Trigger the workflow on push or pull request
-on:
-  push:
-    branches:
-      - '**'
-    tags:
-      - '**'
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        default: true
-
-jobs:
-  build-image:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runs-on: ubuntu-24.04
-          - platform: linux/386
-            runs-on: ubuntu-24.04
-          - platform: linux/arm64
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v7
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v6
-            runs-on: ubuntu-24.04-arm
-
-    name: Build Docker Image for ${{ matrix.platform }}
-    runs-on: ${{ matrix.runs-on }}
-
-    steps:
-      - name: Free Space
-        shell: bash
-        run: |
-          df -h .
-          # Remove android SDK
-          sudo rm -rf /usr/local/lib/android || true
-          # Remove .net runtime
-          sudo rm -rf /usr/share/dotnet || true
-          df -h .
-
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Set PLATFORM Variable
-        run: |
-          platform=${{ matrix.platform }}
-          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Set CACHE_NAME Variable
-        shell: python
-        run: |
-          import os, re
-
-          def slugify(input_string, max_length=63):
-            slug = input_string.lower()
-            slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
-            slug = slug.strip()
-            slug = re.sub(r'\s+', '-', slug)
-            slug = re.sub(r'-+', '-', slug)
-            slug = slug[:max_length]
-            slug = re.sub(r'[-]+$', '', slug)
-            return slug
-
-          ref_name_slug = "cache"
-
-          if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
-            ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"CACHE_NAME={ref_name_slug}\n")
-
-      - name: Get ImageOS
-        # There's no way around this, because "ImageOS" is only available to
-        # processes, but the setup-go action uses it in its key.
-        id: imageos
-        uses: actions/github-script@v7
-        with:
-          result-encoding: string
-          script: |
-            return process.env.ImageOS
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
-        with:
-          images: |
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Load Go Build Cache for Docker
-        id: go-cache
-        uses: actions/cache@v4
-        with:
-          key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          # Cache only the go builds, the module download is cached via the docker layer caching
-          path: |
-            go-build-cache
-
-      - name: Inject Go Build Cache into Docker
-        uses: reproducible-containers/buildkit-cache-dance@v3
-        with:
-          cache-map: |
-            {
-              "go-build-cache": "/root/.cache/go-build"
-            }
-          skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and Publish Image Digest
-        id: build
-        uses: docker/build-push-action@v6
-        with:
-          file: Dockerfile
-          context: .
-          provenance: false
-          # don't specify 'tags' here (error "get can't push tagged ref by digest")
-          # tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          annotations: ${{ steps.meta.outputs.annotations }}
-          platforms: ${{ matrix.platform }}
-          outputs: |
-            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          cache-to: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
-
-      - name: Export Image Digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload Image Digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM }}
-          path: /tmp/digests/*
-          retention-days: 1
-          if-no-files-found: error
-
-  merge-image:
-    name: Merge & Push Final Docker Image
-    runs-on: ubuntu-24.04
-    needs:
-      - build-image
-
-    steps:
-      - name: Download Image Digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digests-*
-          merge-multiple: true
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
-        with:
-          images: |
-            ${{ env.REPO_NAME }}
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Extract Tags
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
-          tags_string = " ".join(tags)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"TAGS={tags_string}\n")
-
-      - name: Extract Annotations
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
-          annotations_string = " ".join(annotations)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"ANNOTATIONS={annotations_string}\n")
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create & Push Manifest List
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create \
-          ${{ env.TAGS }} \
-          ${{ env.ANNOTATIONS }} \
-          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
-
-      - name: Inspect and Run Multi-Platform Image
-        run: |
-          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_docker_plugin.yml

@@ -1,49 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*-
-
-name: Release Build for Docker Plugin
-
-on:
-  release:
-    types: [published]
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        default: true
-
-jobs:
-  build_docker_volume_plugin:
-    if: inputs.manual || github.repository == 'rclone/rclone'
-    name: Build docker plugin job
-    runs-on: ubuntu-latest
-    steps:
-      - name: Free some space
-        shell: bash
-        run: |
-          df -h .
-          # Remove android SDK
-          sudo rm -rf /usr/local/lib/android || true
-          # Remove .net runtime
-          sudo rm -rf /usr/share/dotnet || true
-          df -h .
-      - name: Checkout master
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and publish docker plugin
-        shell: bash
-        run: |
-          VER=${GITHUB_REF#refs/tags/}
-          PLUGIN_USER=rclone
-          docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                       --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-          for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-              export PLUGIN_USER PLUGIN_ARCH
-              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-          done
-          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,89 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKER_HUB_USER }}
+                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+            - name: Build and publish image
+              uses: docker/build-push-action@v6
+              with:
+                file: Dockerfile
+                context: .
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                push: true
+                tags: |
+                  rclone/rclone:latest
+                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
+                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
+                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+
+    build_docker_volume_plugin:
+        if: github.repository == 'rclone/rclone'
+        needs: build
+        runs-on: ubuntu-latest
+        name: Build docker plugin job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Build and publish docker plugin
+              shell: bash
+              run: |
+                VER=${GITHUB_REF#refs/tags/}
+                PLUGIN_USER=rclone
+                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+                    export PLUGIN_USER PLUGIN_ARCH
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+                done
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

Dockerfile

@@ -1,46 +1,18 @@
 FROM golang:alpine AS builder
 
-ARG CGO_ENABLED=0
-
+COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN echo "**** Set Go Environment Variables ****" && \
-    go env -w GOCACHE=/root/.cache/go-build
-
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        make \
-        bash \
-        gawk \
-        git
-
-COPY go.mod .
-COPY go.sum .
-
-RUN echo "**** Download Go Dependencies ****" && \
-    go mod download -x
-
-RUN echo "**** Verify Go Dependencies ****" && \
-    go mod verify
-
-COPY . .
-
-RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
-    echo "**** Build Binary ****" && \
+RUN apk add --no-cache make bash gawk git
+RUN \
+  CGO_ENABLED=0 \
   make
-
-RUN echo "**** Print Version Binary ****" && \
-    ./rclone version
+RUN ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        ca-certificates \
-        fuse3 \
-        tzdata && \
-    echo "Enable user_allow_other in fuse" && \
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

README.md

@@ -1,4 +1,20 @@
-
+<div align="center">
+<sup>Special thanks to our sponsor:</sup>
+<br>
+<br>
+<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
+  <div>
+    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
+  </div>
+  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
+  <div>
+    <sup>Visit warp.dev to learn more.</sup>
+  </div>
+</a>
+<br>
+<hr>
+</div>
+<br>
 
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)

RELEASE.md

@@ -86,16 +86,6 @@ build.
 Once it compiles locally, push it on a test branch and commit fixes
 until the tests pass.
 
-### Major versions
-
-The above procedure will not upgrade major versions, so v2 to v3.
-However this tool can show which major versions might need to be
-upgraded:
-
-    go run github.com/icholy/gomajor@latest list -major
-
-Expect API breakage when updating major versions.
-
 ## Tidy beta
 
 At some point after the release run
@@ -124,8 +114,8 @@ Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * make startstable
   * Do the steps as above
+  * make startstable
   * git co master
   * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

VERSION

@@ -1 +1 @@
-v1.69.2
+v1.69.0

backend/all/all.go

@@ -10,7 +10,6 @@ import (
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/cloudinary"
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"

backend/azureblob/azureblob.go

@@ -2162,9 +2162,6 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/azurefiles/azurefiles.go

@@ -393,10 +393,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 	policyClientOptions := policy.ClientOptions{
 		Transport: newTransporter(ctx),
 	}
-	backup := service.ShareTokenIntentBackup
 	clientOpt := service.ClientOptions{
 		ClientOptions: policyClientOptions,
-		FileRequestIntent: &backup,
 	}
 
 	// Here we auth by setting one of cred, sharedKeyCred or f.client
@@ -899,7 +897,7 @@ func (o *Object) getMetadata(ctx context.Context) error {
 
 // Hash returns the MD5 of an object returning a lowercase hex string
 //
-// May make a network request because the [fs.List] method does not
+// May make a network request becaue the [fs.List] method does not
 // return MD5 hashes for DirEntry
 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
 	if ty != hash.MD5 {

backend/b2/api/types.go

@@ -44,7 +44,6 @@ type Bucket struct {
 type LifecycleRule struct {
 	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
 	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
-	DaysFromStartingToCancelingUnfinishedLargeFiles *int   `json:"daysFromStartingToCancelingUnfinishedLargeFiles"`
 	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 

backend/b2/b2.go

@@ -306,6 +306,7 @@ type Object struct {
 	sha1     string            // SHA-1 hash if known
 	size     int64             // Size of the object
 	mimeType string            // Content-Type of the object
+	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
 }
 
 // ------------------------------------------------------------
@@ -1597,6 +1598,9 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if err != nil {
 		return err
 	}
+	// For now, just set "mtime" in metadata
+	o.meta = make(map[string]string, 1)
+	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
 	return nil
 }
 
@@ -1876,6 +1880,13 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		Info:            Info,
 	}
 
+	// Embryonic metadata support - just mtime
+	o.meta = make(map[string]string, 1)
+	modTime, err := parseTimeStringHelper(info.Info[timeKey])
+	if err == nil {
+		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
+	}
+
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -2220,7 +2231,6 @@ This will dump something like this showing the lifecycle rules.
         {
             "daysFromHidingToDeleting": 1,
             "daysFromUploadingToHiding": null,
-            "daysFromStartingToCancelingUnfinishedLargeFiles": null,
             "fileNamePrefix": ""
         }
     ]
@@ -2249,7 +2259,6 @@ See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 	Opts: map[string]string{
 		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
 		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
-		"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
 	},
 }
 
@@ -2269,13 +2278,6 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 		}
 		newRule.DaysFromUploadingToHiding = &days
 	}
-	if daysStr := opt["daysFromStartingToCancelingUnfinishedLargeFiles"]; daysStr != "" {
-		days, err := strconv.Atoi(daysStr)
-		if err != nil {
-			return nil, fmt.Errorf("bad daysFromStartingToCancelingUnfinishedLargeFiles: %w", err)
-		}
-		newRule.DaysFromStartingToCancelingUnfinishedLargeFiles = &days
-	}
 	bucketName, _ := f.split("")
 	if bucketName == "" {
 		return nil, errors.New("bucket required")
@@ -2283,7 +2285,7 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 	}
 
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err

backend/b2/b2_internal_test.go

@@ -256,6 +256,12 @@ func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string
 			assert.Equal(t, v, got, k)
 		}
 
+		// mtime
+		for k, v := range metadata {
+			got := o.meta[k]
+			assert.Equal(t, v, got, k)
+		}
+
 		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
 
 		// Modification time from the x-bz-info-src_last_modified_millis header

backend/box/box.go

@@ -255,9 +255,6 @@ func getQueryParams(boxConfig *api.ConfigJSON) map[string]string {
 
 func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) {
 	block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey))
-	if block == nil {
-		return nil, errors.New("box: failed to PEM decode private key")
-	}
 	if len(rest) > 0 {
 		return nil, fmt.Errorf("box: extra data included in private key: %w", err)
 	}

backend/chunker/chunker.go

@@ -2480,7 +2480,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	if len(data) > maxMetadataSizeWritten {
 		return nil, false, ErrMetaTooBig
 	}
-	if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
+	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
 		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON

backend/cloudinary/api/types.go

@@ -1,48 +0,0 @@
-// Package api has type definitions for cloudinary
-package api
-
-import (
-	"fmt"
-)
-
-// CloudinaryEncoder extends the built-in encoder
-type CloudinaryEncoder interface {
-	// FromStandardPath takes a / separated path in Standard encoding
-	// and converts it to a / separated path in this encoding.
-	FromStandardPath(string) string
-	// FromStandardName takes name in Standard encoding and converts
-	// it in this encoding.
-	FromStandardName(string) string
-	// ToStandardPath takes a / separated path in this encoding
-	// and converts it to a / separated path in Standard encoding.
-	ToStandardPath(string) string
-	// ToStandardName takes name in this encoding and converts
-	// it in Standard encoding.
-	ToStandardName(string) string
-	// Encoded root of the remote (as passed into NewFs)
-	FromStandardFullPath(string) string
-}
-
-// UpdateOptions was created to pass options from Update to Put
-type UpdateOptions struct {
-	PublicID     string
-	ResourceType string
-	DeliveryType string
-	AssetFolder  string
-	DisplayName  string
-}
-
-// Header formats the option as a string
-func (o *UpdateOptions) Header() (string, string) {
-	return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}
-
-// Mandatory returns whether the option must be parsed or can be ignored
-func (o *UpdateOptions) Mandatory() bool {
-	return false
-}
-
-// String formats the option into human-readable form
-func (o *UpdateOptions) String() string {
-	return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}

backend/cloudinary/cloudinary.go

@@ -1,711 +0,0 @@
-// Package cloudinary provides an interface to the Cloudinary DAM
-package cloudinary
-
-import (
-	"context"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cloudinary/cloudinary-go/v2"
-	SDKApi "github.com/cloudinary/cloudinary-go/v2/api"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin/search"
-	"github.com/cloudinary/cloudinary-go/v2/api/uploader"
-	"github.com/rclone/rclone/backend/cloudinary/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-	"github.com/zeebo/blake3"
-)
-
-// Cloudinary shouldn't have a trailing dot if there is no path
-func cldPathDir(somePath string) string {
-	if somePath == "" || somePath == "." {
-		return somePath
-	}
-	dir := path.Dir(somePath)
-	if dir == "." {
-		return ""
-	}
-	return dir
-}
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "cloudinary",
-		Description: "Cloudinary",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:      "cloud_name",
-				Help:      "Cloudinary Environment Name",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_key",
-				Help:      "Cloudinary API Key",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_secret",
-				Help:      "Cloudinary API Secret",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name: "upload_prefix",
-				Help: "Specify the API endpoint for environments out of the US",
-			},
-			{
-				Name: "upload_preset",
-				Help: "Upload Preset to select asset manipulation on upload",
-			},
-			{
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
-					encoder.EncodeSlash |
-					encoder.EncodeLtGt |
-					encoder.EncodeDoubleQuote |
-					encoder.EncodeQuestion |
-					encoder.EncodeAsterisk |
-					encoder.EncodePipe |
-					encoder.EncodeHash |
-					encoder.EncodePercent |
-					encoder.EncodeBackSlash |
-					encoder.EncodeDel |
-					encoder.EncodeCtl |
-					encoder.EncodeRightSpace |
-					encoder.EncodeInvalidUtf8 |
-					encoder.EncodeDot),
-			},
-			{
-				Name:     "eventually_consistent_delay",
-				Default:  fs.Duration(0),
-				Advanced: true,
-				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",
-			},
-		},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	CloudName                 string               `config:"cloud_name"`
-	APIKey                    string               `config:"api_key"`
-	APISecret                 string               `config:"api_secret"`
-	UploadPrefix              string               `config:"upload_prefix"`
-	UploadPreset              string               `config:"upload_preset"`
-	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EventuallyConsistentDelay fs.Duration          `config:"eventually_consistent_delay"`
-}
-
-// Fs represents a remote cloudinary server
-type Fs struct {
-	name     string
-	root     string
-	opt      Options
-	features *fs.Features
-	pacer    *fs.Pacer
-	srv      *rest.Client           // For downloading assets via the Cloudinary CDN
-	cld      *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK
-	lastCRUD time.Time
-}
-
-// Object describes a cloudinary object
-type Object struct {
-	fs           *Fs
-	remote       string
-	size         int64
-	modTime      time.Time
-	url          string
-	md5sum       string
-	publicID     string
-	resourceType string
-	deliveryType string
-}
-
-// NewFs constructs an Fs from the path, bucket:path
-func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize the Cloudinary client
-	cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)
-	}
-	cld.Admin.Client = *fshttp.NewClient(ctx)
-	cld.Upload.Client = *fshttp.NewClient(ctx)
-	if opt.UploadPrefix != "" {
-		cld.Config.API.UploadPrefix = opt.UploadPrefix
-	}
-	client := fshttp.NewClient(ctx)
-	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		cld:   cld,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
-		srv:   rest.NewClient(client),
-	}
-
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-	}).Fill(ctx, f)
-
-	if root != "" {
-		// Check to see if the root actually an existing file
-		remote := path.Base(root)
-		f.root = cldPathDir(root)
-		_, err := f.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
-				// File doesn't exist so return the previous root
-				f.root = root
-				return f, nil
-			}
-			return nil, err
-		}
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-// ------------------------------------------------------------
-
-// FromStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")
-}
-
-// FromStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
-}
-
-// ToStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")
-}
-
-// ToStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
-}
-
-// FromStandardFullPath encodes a full path to Cloudinary standard
-func (f *Fs) FromStandardFullPath(dir string) string {
-	return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))
-}
-
-// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK
-func (f *Fs) ToAssetFolderAPI(dir string) string {
-	return strings.ReplaceAll(dir, "%", "%25")
-}
-
-// ToDisplayNameElastic encodes a special case of elasticsearch
-func (f *Fs) ToDisplayNameElastic(dir string) string {
-	return strings.ReplaceAll(dir, "!", "\\!")
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// WaitEventuallyConsistent waits till the FS is eventually consistent
-func (f *Fs) WaitEventuallyConsistent() {
-	if f.opt.EventuallyConsistentDelay == fs.Duration(0) {
-		return
-	}
-	delay := time.Duration(f.opt.EventuallyConsistentDelay)
-	timeSinceLastCRUD := time.Since(f.lastCRUD)
-	if timeSinceLastCRUD < delay {
-		time.Sleep(delay - timeSinceLastCRUD)
-	}
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("Cloudinary root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// List the objects and directories in dir into entries
-func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
-	remotePrefix := f.FromStandardFullPath(dir)
-	if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {
-		remotePrefix += "/"
-	}
-
-	var entries fs.DirEntries
-	dirs := make(map[string]struct{})
-	nextCursor := ""
-	f.WaitEventuallyConsistent()
-	for {
-		// user the folders api to list folders.
-		folderParams := admin.SubFoldersParams{
-			Folder:     f.ToAssetFolderAPI(remotePrefix),
-			MaxResults: 500,
-		}
-		if nextCursor != "" {
-			folderParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list sub-folders: %w", err)
-		}
-		if results.Error.Message != "" {
-			if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {
-				return nil, fs.ErrorDirNotFound
-			}
-
-			return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)
-		}
-
-		for _, folder := range results.Folders {
-			relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))
-			parts := strings.Split(relativePath, "/")
-
-			// It's a directory
-			dirName := parts[len(parts)-1]
-			if _, found := dirs[dirName]; !found {
-				d := fs.NewDir(path.Join(dir, dirName), time.Time{})
-				entries = append(entries, d)
-				dirs[dirName] = struct{}{}
-			}
-		}
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	for {
-		// Use the assets.AssetsByAssetFolder API to list assets
-		assetsParams := admin.AssetsByAssetFolderParams{
-			AssetFolder: remotePrefix,
-			MaxResults:  500,
-		}
-		if nextCursor != "" {
-			assetsParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list assets: %w", err)
-		}
-
-		for _, asset := range results.Assets {
-			remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
-			if dir != "" {
-				remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
-			}
-			o := &Object{
-				fs:           f,
-				remote:       remote,
-				size:         int64(asset.Bytes),
-				modTime:      asset.CreatedAt,
-				url:          asset.SecureURL,
-				publicID:     asset.PublicID,
-				resourceType: asset.AssetType,
-				deliveryType: asset.Type,
-			}
-			entries = append(entries, o)
-		}
-
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	return entries, nil
-}
-
-// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	searchParams := search.Query{
-		Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",
-			f.FromStandardFullPath(cldPathDir(remote)),
-			f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),
-		SortBy:     []search.SortByField{{"uploaded_at": "desc"}},
-		MaxResults: 2,
-	}
-	var results *admin.SearchResult
-	f.WaitEventuallyConsistent()
-	err := f.pacer.Call(func() (bool, error) {
-		var err1 error
-		results, err1 = f.cld.Admin.Search(ctx, searchParams)
-		if err1 == nil && results.TotalCount != len(results.Assets) {
-			err1 = errors.New("partial response so waiting for eventual consistency")
-		}
-		return shouldRetry(ctx, nil, err1)
-	})
-	if err != nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-	if results.TotalCount == 0 || len(results.Assets) == 0 {
-		return nil, fs.ErrorObjectNotFound
-	}
-	asset := results.Assets[0]
-
-	o := &Object{
-		fs:           f,
-		remote:       remote,
-		size:         int64(asset.Bytes),
-		modTime:      asset.UploadedAt,
-		url:          asset.SecureURL,
-		md5sum:       asset.Etag,
-		publicID:     asset.PublicID,
-		resourceType: asset.ResourceType,
-		deliveryType: asset.Type,
-	}
-
-	return o, nil
-}
-
-func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {
-	payload := []byte(path.Join(assetFolder, displayName))
-	hash := blake3.Sum256(payload)
-	return hex.EncodeToString(hash[:])
-}
-
-// Put uploads content to Cloudinary
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if src.Size() == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	params := uploader.UploadParams{
-		UploadPreset: f.opt.UploadPreset,
-	}
-
-	updateObject := false
-	var modTime time.Time
-	for _, option := range options {
-		if updateOptions, ok := option.(*api.UpdateOptions); ok {
-			if updateOptions.PublicID != "" {
-				updateObject = true
-				params.Overwrite = SDKApi.Bool(true)
-				params.Invalidate = SDKApi.Bool(true)
-				params.PublicID = updateOptions.PublicID
-				params.ResourceType = updateOptions.ResourceType
-				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
-				params.AssetFolder = updateOptions.AssetFolder
-				params.DisplayName = updateOptions.DisplayName
-				modTime = src.ModTime(ctx)
-			}
-		}
-	}
-	if !updateObject {
-		params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))
-		params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))
-		// We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).
-		// We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.
-		// Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.
-		params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))
-	}
-	uploadResult, err := f.cld.Upload.Upload(ctx, in, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)
-	}
-	if !updateObject {
-		modTime = uploadResult.CreatedAt
-	}
-	if uploadResult.Error.Message != "" {
-		return nil, errors.New(uploadResult.Error.Message)
-	}
-
-	o := &Object{
-		fs:           f,
-		remote:       src.Remote(),
-		size:         int64(uploadResult.Bytes),
-		modTime:      modTime,
-		url:          uploadResult.SecureURL,
-		md5sum:       uploadResult.Etag,
-		publicID:     uploadResult.PublicID,
-		resourceType: uploadResult.ResourceType,
-		deliveryType: uploadResult.Type,
-	}
-	return o, nil
-}
-
-// Precision of the remote
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash sets
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// Mkdir creates empty folders
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}
-	res, err := f.cld.Admin.CreateFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// Rmdir deletes empty folders
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	// Additional test because Cloudinary will delete folders without
-	// assets, regardless of empty sub-folders
-	folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))
-	folderParams := admin.SubFoldersParams{
-		Folder:     folder,
-		MaxResults: 1,
-	}
-	results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-	if err != nil {
-		return err
-	}
-	if results.TotalCount > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-
-	params := admin.DeleteFolderParams{Folder: folder}
-	res, err := f.cld.Admin.DeleteFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {
-			return fs.ErrorDirNotFound
-		}
-
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	420, // Too Many Requests (legacy)
-	429, // Too Many Requests
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	if err != nil {
-		tryAgain := "Try again on "
-		if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
-			layout := "2006-01-02 15:04:05 UTC"
-			dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
-			timestamp, err2 := time.Parse(layout, dateStr)
-			if err2 == nil {
-				return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
-			}
-		}
-
-		fs.Debugf(nil, "Retrying API error %v", err)
-		return true, err
-	}
-
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-// ------------------------------------------------------------
-
-// Hash returns the MD5 of an object
-func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
-	if ty != hash.MD5 {
-		return "", hash.ErrUnsupported
-	}
-	return o.md5sum, nil
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the object
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// Size of object in bytes
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// Storable returns if this object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-// Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: o.url,
-		Options: options,
-	}
-	var offset int64
-	var count int64
-	var key string
-	var value string
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-			key, value = option.Header()
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size - offset
-			key, value = option.Header()
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if key != "" && value != "" {
-		opts.ExtraHeaders = make(map[string]string)
-		opts.ExtraHeaders[key] = value
-	}
-	// Make sure that the asset is fully available
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
-		if err == nil {
-			cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))
-			if clErr == nil && count == int64(cl) {
-				return false, nil
-			}
-		}
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)
-	}
-	return resp.Body, err
-}
-
-// Update the object with the contents of the io.Reader
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	options = append(options, &api.UpdateOptions{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		DeliveryType: o.deliveryType,
-		DisplayName:  api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),
-		AssetFolder:  o.fs.FromStandardFullPath(cldPathDir(o.Remote())),
-	})
-	updatedObj, err := o.fs.Put(ctx, in, src, options...)
-	if err != nil {
-		return err
-	}
-	if uo, ok := updatedObj.(*Object); ok {
-		o.size = uo.size
-		o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time
-		o.url = uo.url
-		o.md5sum = uo.md5sum
-		o.publicID = uo.publicID
-		o.resourceType = uo.resourceType
-		o.deliveryType = uo.deliveryType
-	}
-	return nil
-}
-
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	params := uploader.DestroyParams{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		Type:         o.deliveryType,
-	}
-	res, dErr := o.fs.cld.Upload.Destroy(ctx, params)
-	o.fs.lastCRUD = time.Now()
-	if dErr != nil {
-		return dErr
-	}
-
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	if res.Result != "ok" {
-		return errors.New(res.Result)
-	}
-
-	return nil
-}

backend/cloudinary/cloudinary_test.go

@@ -1,23 +0,0 @@
-// Test Cloudinary filesystem interface
-
-package cloudinary_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/cloudinary"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	name := "TestCloudinary"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      name + ":",
-		NilObject:       (*cloudinary.Object)(nil),
-		SkipInvalidUTF8: true,
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "eventually_consistent_delay", Value: "7"},
-		},
-	})
-}

backend/drive/drive.go

@@ -203,6 +203,7 @@ func driveScopesContainsAppFolder(scopes []string) bool {
 		if scope == scopePrefix+"drive.appfolder" {
 			return true
 		}
+
 	}
 	return false
 }
@@ -1211,7 +1212,6 @@ func fixMimeType(mimeTypeIn string) string {
 	}
 	return mimeTypeOut
 }
-
 func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	out = make(map[string][]string, len(in))
 	for k, v := range in {
@@ -1222,11 +1222,9 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	}
 	return out
 }
-
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
 }
-
 func isLinkMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/x-link-")
 }
@@ -1659,8 +1657,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	ctx context.Context, remote string, info *drive.File,
-	extension, exportName, exportMimeType string, isDocument bool,
-) (o fs.Object, err error) {
+	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
@@ -1763,7 +1760,7 @@ func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Met
 	}
 	var updateMetadata updateMetadataFn
 	if len(metadata) > 0 {
-		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true, true)
+		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true)
 		if err != nil {
 			return nil, fmt.Errorf("create dir: failed to update metadata: %w", err)
 		}
@@ -1794,7 +1791,7 @@ func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata)
 	}
 	dirID = actualID(dirID)
 	updateInfo := &drive.File{}
-	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true, true)
+	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true)
 	if err != nil {
 		return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err)
 	}
@@ -1851,7 +1848,6 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
-
 func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
@@ -1897,8 +1893,7 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
 func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
-	extension, mimeType string, isDocument bool,
-) {
+	extension, mimeType string, isDocument bool) {
 	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
@@ -2694,7 +2689,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if shortcutID != "" {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
-	trashedFiles := false
+	var trashedFiles = false
 	if check {
 		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
 			if !item.Trashed {
@@ -2931,6 +2926,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
+
 	if err != nil {
 		return err
 	}
@@ -3191,7 +3187,6 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
-
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
@@ -3995,13 +3990,14 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	case "query":
 		if len(arg) == 1 {
 			query := arg[0]
-			results, err := f.query(ctx, query)
+			var results, err = f.query(ctx, query)
 			if err != nil {
 				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
 			}
 			return results, nil
-		}
+		} else {
 			return nil, errors.New("need a query argument")
+		}
 	case "rescue":
 		dirID := ""
 		_, delete := opt["delete"]
@@ -4061,7 +4057,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	return "", hash.ErrUnsupported
 }
-
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
 		return "", hash.ErrUnsupported
@@ -4076,8 +4071,7 @@ func (o *baseObject) Size() int64 {
 
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
-	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error,
-) {
+	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -4290,13 +4284,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	return o.baseObject.open(ctx, o.url, options...)
 }
-
 func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
 	var offset, end int64 = 0, -1
-	newOptions := options[:0]
+	var newOptions = options[:0]
 	for _, o := range options {
 		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
@@ -4323,10 +4316,9 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
 	}
 	return
 }
-
 func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
-	data := o.content
+	var data = o.content
 	for _, option := range options {
 		switch x := option.(type) {
 		case *fs.SeekOption:
@@ -4351,8 +4343,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
-	src fs.ObjectInfo,
-) (info *drive.File, err error) {
+	src fs.ObjectInfo) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
 	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
@@ -4430,7 +4421,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	return nil
 }
-
 func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	srcMimeType := fs.MimeType(ctx, src)
 	importMimeType := ""
@@ -4526,7 +4516,6 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
-
 func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }

backend/drive/metadata.go

@@ -508,7 +508,7 @@ type updateMetadataFn func(context.Context, *drive.File) error
 //
 // It returns a callback which should be called to finish the updates
 // after the data is uploaded.
-func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update, isFolder bool) (callback updateMetadataFn, err error) {
+func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) {
 	callbackFns := []updateMetadataFn{}
 	callback = func(ctx context.Context, info *drive.File) error {
 		for _, fn := range callbackFns {
@@ -533,9 +533,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 		}
 		switch k {
 		case "copy-requires-writer-permission":
-			if isFolder {
-				fs.Debugf(f, "Ignoring %s=%s as can't set on folders", k, v)
-			} else if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
+			if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
 				return nil, err
 			}
 		case "writers-can-share":
@@ -632,7 +630,7 @@ func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, opti
 	if err != nil {
 		return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
 	}
-	callback, err = f.updateMetadata(ctx, updateInfo, meta, update, false)
+	callback, err = f.updateMetadata(ctx, updateInfo, meta, update)
 	if err != nil {
 		return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
 	}

backend/dropbox/batcher.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fs/fserrors"
 )
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
@@ -20,10 +21,14 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
 			}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		}
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/dropbox/dropbox.go

@@ -318,46 +318,32 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Some specific errors which should be excluded from retries
-func shouldRetryExclude(ctx context.Context, err error) (bool, error) {
-	if err == nil {
-		return false, err
-	}
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	// First check for specific errors
-	//
-	// These come back from the SDK in a whole host of different
-	// error types, but there doesn't seem to be a consistent way
-	// of reading the error cause, so here we just check using the
-	// error string which isn't perfect but does the job.
+	if err == nil {
+		return false, err
+	}
 	errString := err.Error()
+	// First check for specific errors
 	if strings.Contains(errString, "insufficient_space") {
 		return false, fserrors.FatalError(err)
 	} else if strings.Contains(errString, "malformed_path") {
 		return false, fserrors.NoRetryError(err)
 	}
-	return true, err
-}
-
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if retry, err := shouldRetryExclude(ctx, err); !retry {
-		return retry, err
-	}
 	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
-			fs.Logf(nil, "Error %v. Too many requests or write operations. Trying again in %d seconds.", err, e.RateLimitError.RetryAfter)
+			fs.Logf(errString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
 			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
 	// Keep old behavior for backward compatibility
-	errString := err.Error()
 	if strings.Contains(errString, "too_many_write_operations") || strings.Contains(errString, "too_many_requests") || errString == "" {
 		return true, err
 	}
@@ -1174,16 +1160,6 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		return shouldRetry(ctx, err)
 	})
 
-	if err != nil && createArg.Settings.Expires != nil && strings.Contains(err.Error(), sharing.SharedLinkSettingsErrorNotAuthorized) {
-		// Some plans can't create links with expiry
-		fs.Debugf(absPath, "can't create link with expiry, trying without")
-		createArg.Settings.Expires = nil
-		err = f.pacer.Call(func() (bool, error) {
-			linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
-			return shouldRetry(ctx, err)
-		})
-	}
-
 	if err != nil && strings.Contains(err.Error(),
 		sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
 		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
@@ -1724,10 +1700,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
 			}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		}
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/googlephotos/googlephotos.go

@@ -1168,7 +1168,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		errors := make([]error, 1)
 		results := make([]*api.MediaItem, 1)
 		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
-		if err == nil {
+		if err != nil {
 			err = errors[0]
 			info = results[0]
 		}

backend/googlephotos/googlephotos_test.go

@@ -2,7 +2,6 @@ package googlephotos
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,7 +35,7 @@ func TestIntegration(t *testing.T) {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
-	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
+	if err == fs.ErrorNotFoundInConfigFile {
 		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)

backend/http/http.go

@@ -180,6 +180,7 @@ func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Op
 	}
 	addHeaders(req, opt)
 	res, err := noRedir.Do(req)
+
 	if err != nil {
 		fs.Debugf(nil, "Assuming path is a file as HEAD request could not be sent: %v", err)
 		return createFileResult()
@@ -248,14 +249,6 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	f.httpClient = client
 	f.endpoint = u
 	f.endpointURL = u.String()
-
-	if isFile {
-		// Correct root if definitely pointing to a file
-		f.root = path.Dir(f.root)
-		if f.root == "." || f.root == "/" {
-			f.root = ""
-		}
-	}
 	return isFile, nil
 }
 
@@ -338,13 +331,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
-	trimmedRemote := strings.TrimLeft(remote, "/") // remove leading "/" since we always have it in f.endpointURL
 	if f.opt.NoEscape {
 		// Directly concatenate without escaping, no_escape behavior
-		return f.endpointURL + trimmedRemote
+		return f.endpointURL + remote
 	}
 	// Default behavior
-	return f.endpointURL + rest.URLPathEscape(trimmedRemote)
+	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
 // Errors returned by parseName

backend/http/http_internal_test.go

@@ -191,33 +191,6 @@ func TestNewObject(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
-func TestNewObjectWithLeadingSlash(t *testing.T) {
-	f := prepare(t)
-
-	o, err := f.NewObject(context.Background(), "/four/under four.txt")
-	require.NoError(t, err)
-
-	assert.Equal(t, "/four/under four.txt", o.Remote())
-	assert.Equal(t, int64(8+lineEndSize), o.Size())
-	_, ok := o.(*Object)
-	assert.True(t, ok)
-
-	// Test the time is correct on the object
-
-	tObj := o.ModTime(context.Background())
-
-	fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
-	require.NoError(t, err)
-	tFile := fi.ModTime()
-
-	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
-
-	// check object not found
-	o, err = f.NewObject(context.Background(), "/not found.txt")
-	assert.Nil(t, o)
-	assert.Equal(t, fs.ErrorObjectNotFound, err)
-}
-
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

backend/iclouddrive/api/drive.go

@@ -631,7 +631,7 @@ func NewUpdateFileInfo() UpdateFileInfo {
 		FileFlags: FileFlags{
 			IsExecutable: true,
 			IsHidden:     false,
-			IsWritable:   true,
+			IsWritable:   false,
 		},
 	}
 }

backend/iclouddrive/iclouddrive.go

@@ -445,7 +445,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// build request
-	// can't use normal rename as file needs to be "activated" first
+	// cant use normal rename as file needs to be "activated" first
 
 	r := api.NewUpdateFileInfo()
 	r.DocumentID = doc.DocumentID

backend/imagekit/client/media.go

@@ -75,7 +75,7 @@ type MoveFolderParam struct {
 	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
 }
 
-// JobIDResponse represents response struct with JobID for folder operations
+// JobIDResponse respresents response struct with JobID for folder operations
 type JobIDResponse struct {
 	JobID string `json:"jobId"`
 }

backend/local/about_windows.go

@@ -5,18 +5,18 @@ package local
 import (
 	"context"
 	"fmt"
+	"syscall"
 	"unsafe"
 
 	"github.com/rclone/rclone/fs"
-	"golang.org/x/sys/windows"
 )
 
-var getFreeDiskSpace = windows.NewLazySystemDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
+var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var available, total, free int64
-	root, e := windows.UTF16PtrFromString(f.root)
+	root, e := syscall.UTF16PtrFromString(f.root)
 	if e != nil {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e)
 	}
@@ -26,7 +26,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
 		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes
 	)
-	if e1 != windows.Errno(0) {
+	if e1 != syscall.Errno(0) {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e1)
 	}
 	usage := &fs.Usage{

backend/local/local.go

@@ -34,6 +34,7 @@ import (
 // Constants
 const (
 	devUnset   = 0xdeadbeefcafebabe                                     // a device id meaning it is unset
+	linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
 	useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
 )
 
@@ -100,8 +101,10 @@ Metadata is supported on files and directories.
 			},
 			{
 				Name:     "links",
-				Help:     "Translate symlinks to/from regular files with a '" + fs.LinkSuffix + "' extension for the local backend.",
+				Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.",
 				Default:  false,
+				NoPrefix: true,
+				ShortOpt: "l",
 				Advanced: true,
 			},
 			{
@@ -376,22 +379,17 @@ type Directory struct {
 
 var (
 	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-	errLinksNeedsSuffix  = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
+	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
 )
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ci := fs.GetConfig(ctx)
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
 	}
-	// Override --local-links with --links if set
-	if ci.Links {
-		opt.TranslateSymlinks = true
-	}
 	if opt.TranslateSymlinks && opt.FollowSymlinks {
 		return nil, errLinksAndCopyLinks
 	}
@@ -437,9 +435,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
 	// Check to see if this is a .rclonelink if not found
-	hasLinkSuffix := strings.HasSuffix(f.root, fs.LinkSuffix)
+	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
 	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
-		fi, err = f.lstat(strings.TrimSuffix(f.root, fs.LinkSuffix))
+		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
 	}
 	if err == nil && f.isRegular(fi.Mode()) {
 		// Handle the odd case, that a symlink was specified by name without the link suffix
@@ -510,8 +508,8 @@ func (f *Fs) caseInsensitive() bool {
 //
 // for regular files, localPath is returned unchanged
 func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
-	newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
+	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
+	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
 	return newLocalPath, isTranslatedLink
 }
 
@@ -694,7 +692,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			} else {
 				// Check whether this link should be translated
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += fs.LinkSuffix
+					newRemote += linkSuffix
 				}
 				// Don't include non directory if not included
 				// we leave directory filtering to the layer above

backend/local/local_internal_test.go

@@ -110,7 +110,7 @@ func TestSymlink(t *testing.T) {
 	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
 
 	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+fs.LinkSuffix, "file.txt", modTime2)
+	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
 
 	// Object viewed as destination
 	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
@@ -139,7 +139,7 @@ func TestSymlink(t *testing.T) {
 
 	// Create a symlink
 	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+fs.LinkSuffix, "file.txt", modTime3, false)
+	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
 	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
 	if haveLChtimes {
 		r.CheckLocalItems(t, file1, file2, file3)
@@ -155,9 +155,9 @@ func TestSymlink(t *testing.T) {
 	assert.Equal(t, "file.txt", linkText)
 
 	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+fs.LinkSuffix)
+	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+linkSuffix)
 	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+fs.LinkSuffix, o.Remote())
+	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
 	assert.Equal(t, int64(8), o.Size())
 
 	// Check that NewObject doesn't see the non suffixed version
@@ -165,7 +165,7 @@ func TestSymlink(t *testing.T) {
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
 	// Check that NewFs works with the suffixed version and --links
-	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+fs.LinkSuffix), configmap.Simple{
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
 		"links": "true",
 	})
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -277,7 +277,7 @@ func TestMetadata(t *testing.T) {
 	// Write a symlink to the file
 	symlinkPath := "metafile-link.txt"
 	osSymlinkPath := filepath.Join(f.root, symlinkPath)
-	symlinkPath += fs.LinkSuffix
+	symlinkPath += linkSuffix
 	require.NoError(t, os.Symlink(filePath, osSymlinkPath))
 	symlinkModTime := fstest.Time("2002-02-03T04:05:10.123123123Z")
 	require.NoError(t, lChtimes(osSymlinkPath, symlinkModTime, symlinkModTime))

backend/onedrive/metadata.go

@@ -396,57 +396,10 @@ func (m *Metadata) WritePermissions(ctx context.Context) (err error) {
 	return nil
 }
 
-// Order the permissions so that any with users come first.
-//
-// This is to work around a quirk with Graph:
-//
-// 1. You are adding permissions for both a group and a user.
-// 2. The user is a member of the group.
-// 3. The permissions for the group and user are the same.
-// 4. You are adding the group permission before the user permission.
-//
-// When all of the above are true, Graph indicates it has added the
-// user permission, but it immediately drops it
-//
-// See: https://github.com/rclone/rclone/issues/8465
-func (m *Metadata) orderPermissions(xs []*api.PermissionsType) {
-	// Return true if identity has any user permissions
-	hasUserIdentity := func(identity *api.IdentitySet) bool {
-		if identity == nil {
-			return false
-		}
-		return identity.User.ID != "" || identity.User.DisplayName != "" || identity.User.Email != "" || identity.User.LoginName != ""
-	}
-	// Return true if p has any user permissions
-	hasUser := func(p *api.PermissionsType) bool {
-		if hasUserIdentity(p.GetGrantedTo(m.fs.driveType)) {
-			return true
-		}
-		for _, identity := range p.GetGrantedToIdentities(m.fs.driveType) {
-			if hasUserIdentity(identity) {
-				return true
-			}
-		}
-		return false
-	}
-	// Put Permissions with a user first, leaving unsorted otherwise
-	slices.SortStableFunc(xs, func(a, b *api.PermissionsType) int {
-		aHasUser := hasUser(a)
-		bHasUser := hasUser(b)
-		if aHasUser && !bHasUser {
-			return -1
-		} else if !aHasUser && bHasUser {
-			return 1
-		}
-		return 0
-	})
-}
-
 // sortPermissions sorts the permissions (to be written) into add, update, and remove queues
 func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType) {
 	new, old := m.queuedPermissions, m.permissions
 	if len(old) == 0 || m.permsAddOnly {
-		m.orderPermissions(new)
 		return new, nil, nil // they must all be "add"
 	}
 
@@ -494,9 +447,6 @@ func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType
 			remove = append(remove, o)
 		}
 	}
-	m.orderPermissions(add)
-	m.orderPermissions(update)
-	m.orderPermissions(remove)
 	return add, update, remove
 }
 

backend/onedrive/metadata_test.go

@@ -1,125 +0,0 @@
-package onedrive
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/rclone/rclone/backend/onedrive/api"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestOrderPermissions(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    []*api.PermissionsType
-		expected []string
-	}{
-		{
-			name:     "empty",
-			input:    []*api.PermissionsType{},
-			expected: []string(nil),
-		},
-		{
-			name: "users first, then group, then none",
-			input: []*api.PermissionsType{
-				{ID: "1", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group1"}}},
-				{ID: "2", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{DisplayName: "Alice"}}}},
-				{ID: "3", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Alice"}}},
-				{ID: "4"},
-			},
-			expected: []string{"2", "3", "1", "4"},
-		},
-		{
-			name: "same type unsorted",
-			input: []*api.PermissionsType{
-				{ID: "b", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group B"}}},
-				{ID: "a", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group A"}}},
-				{ID: "c", GrantedToIdentities: []*api.IdentitySet{{Group: api.Identity{DisplayName: "Group A"}}, {User: api.Identity{DisplayName: "Alice"}}}},
-			},
-			expected: []string{"c", "b", "a"},
-		},
-		{
-			name: "all user identities",
-			input: []*api.PermissionsType{
-				{ID: "c", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Bob"}}},
-				{ID: "a", GrantedTo: &api.IdentitySet{User: api.Identity{Email: "alice@example.com"}}},
-				{ID: "b", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{LoginName: "user3"}}}},
-			},
-			expected: []string{"c", "a", "b"},
-		},
-		{
-			name: "no user or group info",
-			input: []*api.PermissionsType{
-				{ID: "z"},
-				{ID: "x"},
-				{ID: "y"},
-			},
-			expected: []string{"z", "x", "y"},
-		},
-	}
-
-	for _, driveType := range []string{driveTypePersonal, driveTypeBusiness} {
-		t.Run(driveType, func(t *testing.T) {
-			for _, tt := range tests {
-				m := &Metadata{fs: &Fs{driveType: driveType}}
-				t.Run(tt.name, func(t *testing.T) {
-					if driveType == driveTypeBusiness {
-						for i := range tt.input {
-							tt.input[i].GrantedToV2 = tt.input[i].GrantedTo
-							tt.input[i].GrantedTo = nil
-							tt.input[i].GrantedToIdentitiesV2 = tt.input[i].GrantedToIdentities
-							tt.input[i].GrantedToIdentities = nil
-						}
-					}
-					m.orderPermissions(tt.input)
-					var gotIDs []string
-					for _, p := range tt.input {
-						gotIDs = append(gotIDs, p.ID)
-					}
-					assert.Equal(t, tt.expected, gotIDs)
-				})
-			}
-		})
-	}
-}
-
-func TestOrderPermissionsJSON(t *testing.T) {
-	testJSON := `[
-  {
-    "id": "1",
-    "grantedToV2": {
-      "group": {
-        "id": "group@example.com"
-      }
-    },
-    "roles": [
-      "write"
-    ]
-  },
-  {
-    "id": "2",
-    "grantedToV2": {
-      "user": {
-        "id": "user@example.com"
-      }
-    },
-    "roles": [
-      "write"
-    ]
-  }
-]`
-
-	var testPerms []*api.PermissionsType
-	err := json.Unmarshal([]byte(testJSON), &testPerms)
-	require.NoError(t, err)
-
-	m := &Metadata{fs: &Fs{driveType: driveTypeBusiness}}
-	m.orderPermissions(testPerms)
-	var gotIDs []string
-	for _, p := range testPerms {
-		gotIDs = append(gotIDs, p.ID)
-	}
-	assert.Equal(t, []string{"2", "1"}, gotIDs)
-
-}

backend/onedrive/onedrive.go

@@ -131,7 +131,7 @@ func init() {
 					Help:  "Microsoft Cloud for US Government",
 				}, {
 					Value: regionDE,
-					Help:  "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).",
+					Help:  "Microsoft Cloud Germany",
 				}, {
 					Value: regionCN,
 					Help:  "Azure and Office 365 operated by Vnet Group in China",

backend/oracleobjectstorage/multipart.go

@@ -22,7 +22,6 @@ import (
 	"github.com/oracle/oci-go-sdk/v65/objectstorage"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
-	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 )
 
@@ -184,9 +183,6 @@ func (w *objectChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, rea
 			if ossPartNumber <= 8 {
 				return shouldRetry(ctx, resp.HTTPResponse(), err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/pcloud/pcloud.go

@@ -424,7 +424,7 @@ func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) {
 	})
 	// Set our own http client in the context
 	ctx = oauthutil.Context(ctx, baseClient)
-	// create a new oauth client, reuse the token source
+	// create a new oauth client, re-use the token source
 	oAuthClient := oauth2.NewClient(ctx, f.ts)
 	return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil
 }

backend/pikpak/pikpak.go

@@ -213,11 +213,6 @@ Fill in for rclone to use a non root folder as its starting point.
 			Default:  false,
 			Help:     "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
 			Advanced: true,
-		}, {
-			Name:     "no_media_link",
-			Default:  false,
-			Help:     "Use original file links instead of media links.\n\nThis avoids issues caused by invalid media links, but may reduce download speeds.",
-			Advanced: true,
 		}, {
 			Name:     "hash_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate hash if required.",
@@ -291,7 +286,6 @@ type Options struct {
 	RootFolderID        string               `config:"root_folder_id"`
 	UseTrash            bool                 `config:"use_trash"`
 	TrashedOnly         bool                 `config:"trashed_only"`
-	NoMediaLink         bool                 `config:"no_media_link"`
 	HashMemoryThreshold fs.SizeSuffix        `config:"hash_memory_limit"`
 	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
 	UploadConcurrency   int                  `config:"upload_concurrency"`
@@ -1581,18 +1575,19 @@ func (o *Object) setMetaData(info *api.File) (err error) {
 	o.md5sum = info.Md5Checksum
 	if info.Links.ApplicationOctetStream != nil {
 		o.link = info.Links.ApplicationOctetStream
-		if !o.fs.opt.NoMediaLink {
 		if fid := parseFileID(o.link.URL); fid != "" {
-				for _, media := range info.Medias {
-					if media.Link != nil && parseFileID(media.Link.URL) == fid {
-						fs.Debugf(o, "Using a media link")
+			for mid, media := range info.Medias {
+				if media.Link == nil {
+					continue
+				}
+				if mfid := parseFileID(media.Link.URL); fid == mfid {
+					fs.Debugf(o, "Using a media link from Medias[%d]", mid)
 					o.link = media.Link
 					break
 				}
 			}
 		}
 	}
-	}
 	return nil
 }
 

backend/s3/s3.go

@@ -934,67 +934,34 @@ func init() {
 				Help:  "The default endpoint\nIran",
 			}},
 		}, {
-			// Linode endpoints: https://techdocs.akamai.com/cloud-computing/docs/object-storage-product-limits#supported-endpoint-types-by-region
+			// Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint
 			Name:     "endpoint",
 			Help:     "Endpoint for Linode Object Storage API.",
 			Provider: "Linode",
 			Examples: []fs.OptionExample{{
-				Value: "nl-ams-1.linodeobjects.com",
-				Help:  "Amsterdam (Netherlands), nl-ams-1",
-			}, {
 				Value: "us-southeast-1.linodeobjects.com",
 				Help:  "Atlanta, GA (USA), us-southeast-1",
-			}, {
-				Value: "in-maa-1.linodeobjects.com",
-				Help:  "Chennai (India), in-maa-1",
 			}, {
 				Value: "us-ord-1.linodeobjects.com",
 				Help:  "Chicago, IL (USA), us-ord-1",
 			}, {
 				Value: "eu-central-1.linodeobjects.com",
 				Help:  "Frankfurt (Germany), eu-central-1",
-			}, {
-				Value: "id-cgk-1.linodeobjects.com",
-				Help:  "Jakarta (Indonesia), id-cgk-1",
-			}, {
-				Value: "gb-lon-1.linodeobjects.com",
-				Help:  "London 2 (Great Britain), gb-lon-1",
-			}, {
-				Value: "us-lax-1.linodeobjects.com",
-				Help:  "Los Angeles, CA (USA), us-lax-1",
-			}, {
-				Value: "es-mad-1.linodeobjects.com",
-				Help:  "Madrid (Spain), es-mad-1",
-			}, {
-				Value: "au-mel-1.linodeobjects.com",
-				Help:  "Melbourne (Australia), au-mel-1",
-			}, {
-				Value: "us-mia-1.linodeobjects.com",
-				Help:  "Miami, FL (USA), us-mia-1",
 			}, {
 				Value: "it-mil-1.linodeobjects.com",
 				Help:  "Milan (Italy), it-mil-1",
 			}, {
 				Value: "us-east-1.linodeobjects.com",
 				Help:  "Newark, NJ (USA), us-east-1",
-			}, {
-				Value: "jp-osa-1.linodeobjects.com",
-				Help:  "Osaka (Japan), jp-osa-1",
 			}, {
 				Value: "fr-par-1.linodeobjects.com",
 				Help:  "Paris (France), fr-par-1",
-			}, {
-				Value: "br-gru-1.linodeobjects.com",
-				Help:  "São Paulo (Brazil), br-gru-1",
 			}, {
 				Value: "us-sea-1.linodeobjects.com",
 				Help:  "Seattle, WA (USA), us-sea-1",
 			}, {
 				Value: "ap-south-1.linodeobjects.com",
-				Help:  "Singapore, ap-south-1",
-			}, {
-				Value: "sg-sin-1.linodeobjects.com",
-				Help:  "Singapore 2, sg-sin-1",
+				Help:  "Singapore ap-south-1",
 			}, {
 				Value: "se-sto-1.linodeobjects.com",
 				Help:  "Stockholm (Sweden), se-sto-1",
@@ -1376,7 +1343,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1389,10 +1356,6 @@ func init() {
 				Value:    "sfo3.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces San Francisco 3",
 				Provider: "DigitalOcean",
-			}, {
-				Value:    "sfo2.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces San Francisco 2",
-				Provider: "DigitalOcean",
 			}, {
 				Value:    "fra1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Frankfurt 1",
@@ -1409,18 +1372,6 @@ func init() {
 				Value:    "sgp1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
-			}, {
-				Value:    "lon1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces London 1",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "tor1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Toronto 1",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "blr1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Bangalore 1",
-				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
@@ -1525,6 +1476,14 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
+			}, {
+				Value:    "br-se1.magaluobjects.com",
+				Help:     "Magalu BR Southeast 1 endpoint",
+				Provider: "Magalu",
+			}, {
+				Value:    "br-ne1.magaluobjects.com",
+				Help:     "Magalu BR Northeast 1 endpoint",
+				Provider: "Magalu",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -2097,7 +2056,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "One Zone Infrequent Access storage class",
 			}, {
 				Value: "GLACIER",
-				Help:  "Glacier Flexible Retrieval storage class",
+				Help:  "Glacier storage class",
 			}, {
 				Value: "DEEP_ARCHIVE",
 				Help:  "Glacier Deep Archive storage class",
@@ -2163,16 +2122,13 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "Standard storage class",
 			}},
 		}, {
-			// Mapping from here: https://docs.magalu.cloud/docs/storage/object-storage/Classes-de-Armazenamento/standard
+			// Mapping from here: #todo
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in Magalu.",
 			Provider: "Magalu",
 			Examples: []fs.OptionExample{{
 				Value: "STANDARD",
 				Help:  "Standard storage class",
-			}, {
-				Value: "GLACIER_IR",
-				Help:  "Glacier Instant Retrieval storage class",
 			}},
 		}, {
 			// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
@@ -3388,7 +3344,7 @@ func setQuirks(opt *Options) {
 		listObjectsV2         = true // Always use ListObjectsV2 instead of ListObjects
 		virtualHostStyle      = true // Use bucket.provider.com instead of putting the bucket in the URL
 		urlEncodeListings     = true // URL encode the listings to help with control characters
-		useMultipartEtag      = true // Set if Etags for multipart uploads are compatible with AWS
+		useMultipartEtag      = true // Set if Etags for multpart uploads are compatible with AWS
 		useAcceptEncodingGzip = true // Set Accept-Encoding: gzip
 		mightGzip             = true // assume all providers might use content encoding gzip until proven otherwise
 		useAlreadyExists      = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
@@ -4976,7 +4932,7 @@ or from INTELLIGENT-TIERING Archive Access / Deep Archive Access tier to the Fre
 
 Usage Examples:
 
-    rclone backend restore s3:bucket/path/to/ --include /object -o priority=PRIORITY -o lifetime=DAYS
+    rclone backend restore s3:bucket/path/to/object -o priority=PRIORITY -o lifetime=DAYS
     rclone backend restore s3:bucket/path/to/directory -o priority=PRIORITY -o lifetime=DAYS
     rclone backend restore s3:bucket -o priority=PRIORITY -o lifetime=DAYS
     rclone backend restore s3:bucket/path/to/directory -o priority=PRIORITY
@@ -6101,7 +6057,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 			if mOut == nil {
 				err = fserrors.RetryErrorf("internal error: no info from multipart upload")
 			} else if mOut.UploadId == nil {
-				err = fserrors.RetryErrorf("internal error: no UploadId in multipart upload: %#v", *mOut)
+				err = fserrors.RetryErrorf("internal error: no UploadId in multpart upload: %#v", *mOut)
 			}
 		}
 		return f.shouldRetry(ctx, err)
@@ -6219,9 +6175,6 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/smb/smb.go

@@ -601,10 +601,9 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	}
 
 	fi, err := cn.smbShare.Stat(reqDir)
-	if err != nil {
-		return fmt.Errorf("SetModTime: stat: %w", err)
-	}
+	if err == nil {
 		o.statResult = fi
+	}
 	return err
 }
 
@@ -686,6 +685,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 	defer func() {
+		o.statResult, _ = cn.smbShare.Stat(filename)
 		o.fs.putConnection(&cn)
 	}()
 
@@ -723,7 +723,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update Close failed: %w", err)
 	}
 
-	// Set the modified time and also o.statResult
+	// Set the modified time
 	err = o.SetModTime(ctx, src.ModTime(ctx))
 	if err != nil {
 		return fmt.Errorf("Update SetModTime failed: %w", err)

backend/sugarsync/sugarsync.go

@@ -327,7 +327,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
 func (f *Fs) getAuthToken(ctx context.Context) error {
 	fs.Debugf(f, "Renewing token")
 
-	authRequest := api.TokenAuthRequest{
+	var authRequest = api.TokenAuthRequest{
 		AccessKeyID:      withDefault(f.opt.AccessKeyID, accessKeyID),
 		PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)),
 		RefreshToken:     f.opt.RefreshToken,
@@ -509,7 +509,7 @@ func errorHandler(resp *http.Response) (err error) {
 		return fmt.Errorf("error reading error out of body: %w", err)
 	}
 	match := findError.FindSubmatch(body)
-	if len(match) < 2 || len(match[1]) == 0 {
+	if match == nil || len(match) < 2 || len(match[1]) == 0 {
 		return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
 	}
 	return fmt.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1])

backend/webdav/webdav.go

@@ -161,24 +161,7 @@ Set to 0 to disable chunked uploading.
 			Default:  false,
 		},
 			fshttp.UnixSocketConfig,
-			{
-				Name: "auth_redirect",
-				Help: `Preserve authentication on redirect.
-
-If the server redirects rclone to a new domain when it is trying to
-read a file then normally rclone will drop the Authorization: header
-from the request.
-
-This is standard security practice to avoid sending your credentials
-to an unknown webserver.
-
-However this is desirable in some circumstances. If you are getting
-an error like "401 Unauthorized" when rclone is attempting to read
-files from the webdav server then you can try this option.
-`,
-				Advanced: true,
-				Default:  false,
-			}},
+		},
 	})
 }
 
@@ -197,7 +180,6 @@ type Options struct {
 	ExcludeShares      bool                 `config:"owncloud_exclude_shares"`
 	ExcludeMounts      bool                 `config:"owncloud_exclude_mounts"`
 	UnixSocket         string               `config:"unix_socket"`
-	AuthRedirect       bool                 `config:"auth_redirect"`
 }
 
 // Fs represents a remote webdav
@@ -1474,7 +1456,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		ExtraHeaders: map[string]string{
 			"Depth": "0",
 		},
-		AuthRedirect: o.fs.opt.AuthRedirect, // allow redirects to preserve Auth
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)

bin/.ignore-emails

@@ -12,4 +12,3 @@
 <seb•ɑƬ•chezwam•ɖɵʈ•org>
 <allllaboutyou@gmail.com>
 <psycho@feltzv.fr>
-<afw5059@gmail.com>

bin/make_manual.py

@@ -7,7 +7,6 @@ conversion into man pages etc.
 import os
 import re
 import time
-import subprocess
 from datetime import datetime
 
 docpath = "docs/content"
@@ -36,7 +35,6 @@ docs = [
     "box.md",
     "cache.md",
     "chunker.md",
-    "cloudinary.md",
     "sharefile.md",
     "crypt.md",
     "compress.md",
@@ -193,23 +191,13 @@ def main():
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
     build_date = datetime.utcfromtimestamp(
             int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
-    help_output = subprocess.check_output(["rclone", "help"]).decode("utf-8")
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-# NAME
-
-rclone - manage files on cloud storage
-
-# SYNOPSIS
-
-```
-%s
-```
-""" % (build_date.strftime("%b %d, %Y"), help_output))
+""" % build_date.strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

bin/not-in-stable.go

@@ -29,7 +29,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 	cmd := exec.Command("git", "log", "--oneline", from+".."+to)
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	logMap = map[string]string{}
 	logs = []string{}
@@ -39,7 +39,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 		}
 		match := logRe.FindSubmatch(line)
 		if match == nil {
-			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 		}
 		var hash, logMessage = string(match[1]), string(match[2])
 		logMap[logMessage] = hash
@@ -52,12 +52,12 @@ func main() {
 	flag.Parse()
 	args := flag.Args()
 	if len(args) != 0 {
-		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	// v1.54.0
 	versionBytes, err := os.ReadFile("VERSION")
 	if err != nil {
-		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	if versionBytes[0] == 'v' {
 		versionBytes = versionBytes[1:]
@@ -65,7 +65,7 @@ func main() {
 	versionBytes = bytes.TrimSpace(versionBytes)
 	semver := semver.New(string(versionBytes))
 	stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1)
-	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	masterMap, masterLogs := readCommits(stable+".0", "master")
 	stableMap, _ := readCommits(stable+".0", stable+"-stable")
 	for _, logMessage := range masterLogs {

bin/test_backend_sizes.py

@@ -7,18 +7,15 @@ Run with no arguments to test all backends or a supply a list of
 backends to test.
 """
 
-import os
-import re
-import sys
-import subprocess
-
 all_backends = "backend/all/all.go"
 
 # compile command which is more or less like the production builds
 compile_command = ["go", "build", "--ldflags", "-s", "-trimpath"]
 
-# disable CGO as that makes a lot of difference to binary size
-os.environ["CGO_ENABLED"]="0"
+import os
+import re
+import sys
+import subprocess
 
 match_backend = re.compile(r'"github.com/rclone/rclone/backend/(.*?)"')
 
@@ -46,9 +43,6 @@ def write_all(orig_all, backend):
             # Comment out line matching backend
             if match and match.group(1) == backend:
                 line = "// " + line
-            # s3 and pikpak depend on each other
-            if backend == "s3" and "pikpak" in line:
-                line = "// " + line
             fd.write(line+"\n")
 
 def compile():

cmd/authorize/authorize.go

@@ -23,23 +23,19 @@ func init() {
 }
 
 var commandDefinition = &cobra.Command{
-	Use:   "authorize <fs name> [base64_json_blob | client_id client_secret]",
+	Use:   "authorize",
 	Short: `Remote authorization.`,
 	Long: `Remote authorization. Used to authorize a remote or headless
 rclone from a machine with a browser - use as instructed by
 rclone config.
 
-The command requires 1-3 arguments:
-  - fs name (e.g., "drive", "s3", etc.)
-  - Either a base64 encoded JSON blob obtained from a previous rclone config session
-  - Or a client_id and client_secret pair obtained from the remote service
-
 Use --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.
 
 Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.`,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.27",
+		// "groups":            "",
 	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 3, command, args)

cmd/authorize/authorize_test.go

@@ -1,32 +0,0 @@
-package authorize
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/spf13/cobra"
-)
-
-func TestAuthorizeCommand(t *testing.T) {
-	// Test that the Use string is correctly formatted
-	if commandDefinition.Use != "authorize <fs name> [base64_json_blob | client_id client_secret]" {
-		t.Errorf("Command Use string doesn't match expected format: %s", commandDefinition.Use)
-	}
-
-	// Test that help output contains the argument information
-	buf := &bytes.Buffer{}
-	cmd := &cobra.Command{}
-	cmd.AddCommand(commandDefinition)
-	cmd.SetOut(buf)
-	cmd.SetArgs([]string{"authorize", "--help"})
-	err := cmd.Execute()
-	if err != nil {
-		t.Fatalf("Failed to execute help command: %v", err)
-	}
-
-	helpOutput := buf.String()
-	if !strings.Contains(helpOutput, "authorize <fs name>") {
-		t.Errorf("Help output doesn't contain correct usage information")
-	}
-}

cmd/bisync/bisync_test.go

@@ -746,16 +746,6 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 	case "test-func":
 		b.TestFn = testFunc
 		return
-	case "concurrent-func":
-		b.TestFn = func() {
-			src := filepath.Join(b.dataDir, "file7.txt")
-			dst := "file1.txt"
-			err := b.copyFile(ctx, src, b.replaceHex(b.path2), dst)
-			if err != nil {
-				fs.Errorf(src, "error copying file: %v", err)
-			}
-		}
-		return
 	case "fix-names":
 		// in case the local os converted any filenames
 		ci.NoUnicodeNormalization = true
@@ -881,9 +871,10 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 		if !ok || err != nil {
 			fs.Logf(remotePath, "Can't find expected file %s (was it renamed by the os?) %v", args[1], err)
 			return
-		}
+		} else {
 			// include hash of filename to make unicode form differences easier to see in logs
 			fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf))
+		}
 		return
 	default:
 		return fmt.Errorf("unknown command: %q", args[0])

cmd/bisync/compare.go

@@ -218,7 +218,7 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {
 	if b.opt.CompareFlag == "" {
 		return nil
 	}
-	var CompareFlag CompareOpt // for exclusions
+	var CompareFlag CompareOpt // for exlcusions
 	opts := strings.Split(b.opt.CompareFlag, ",")
 	for _, opt := range opts {
 		switch strings.ToLower(strings.TrimSpace(opt)) {

cmd/bisync/deltas.go

@@ -161,7 +161,9 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 		return
 	}
 
+	if err == nil {
 		err = b.checkListing(now, newListing, "current "+msg)
+	}
 	if err != nil {
 		return
 	}
@@ -284,7 +286,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 }
 
 // applyDeltas
-func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (results2to1, results1to2 []Results, queues queues, err error) {
+func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changes1, changes2 bool, results2to1, results1to2 []Results, queues queues, err error) {
 	path1 := bilib.FsPath(b.fs1)
 	path2 := bilib.FsPath(b.fs2)
 
@@ -484,6 +486,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 
 	// Do the batch operation
 	if copy2to1.NotEmpty() && !b.InGracefulShutdown {
+		changes1 = true
 		b.indent("Path2", "Path1", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 1)
 		results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")
@@ -500,6 +503,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 	}
 
 	if copy1to2.NotEmpty() && !b.InGracefulShutdown {
+		changes2 = true
 		b.indent("Path1", "Path2", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 2)
 		results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")

cmd/bisync/listing.go

@@ -394,7 +394,7 @@ func parseHash(str string) (string, string, error) {
 	return "", "", fmt.Errorf("invalid hash %q", str)
 }
 
-// checkListing verifies that listing is not empty (unless resyncing)
+// checkListing verifies that listing is not empty (unless resynching)
 func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error {
 	if b.opt.Resync || !ls.empty() {
 		return nil

cmd/bisync/operations.go

@@ -359,6 +359,8 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 
 	// Determine and apply changes to Path1 and Path2
 	noChanges := ds1.empty() && ds2.empty()
+	changes1 := false // 2to1
+	changes2 := false // 1to2
 	results2to1 := []Results{}
 	results1to2 := []Results{}
 
@@ -368,7 +370,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 		fs.Infof(nil, "No changes found")
 	} else {
 		fs.Infof(nil, "Applying changes")
-		results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
+		changes1, changes2, results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
 		if err != nil {
 			if b.InGracefulShutdown && (err == context.Canceled || err == accounting.ErrorMaxTransferLimitReachedGraceful || strings.Contains(err.Error(), "context canceled")) {
 				fs.Infof(nil, "Ignoring sync error due to Graceful Shutdown: %v", err)
@@ -393,11 +395,21 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 	}
 	b.saveOldListings()
 	// save new listings
+	// NOTE: "changes" in this case does not mean this run vs. last run, it means start of this run vs. end of this run.
+	// i.e. whether we can use the March lst-new as this side's lst without modifying it.
 	if noChanges {
 		b.replaceCurrentListings()
 	} else {
-		err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) // 2to1
-		err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)  // 1to2
+		if changes1 || b.InGracefulShutdown { // 2to1
+			err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false)
+		} else {
+			err1 = bilib.CopyFileIfExists(b.newListing1, b.listing1)
+		}
+		if changes2 || b.InGracefulShutdown { // 1to2
+			err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)
+		} else {
+			err2 = bilib.CopyFileIfExists(b.newListing2, b.listing2)
+		}
 	}
 	if b.DebugName != "" {
 		l1, _ := b.loadListing(b.listing1)

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -1 +0,0 @@
-"file1.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/test.log

@@ -1,73 +0,0 @@
-(01)  : test concurrent
-
-(02)  : test initial bisync
-(03)  : bisync resync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Copying Path2 files to Path1
-INFO  : - Path2    Resync is copying files to         - Path1
-INFO  : - Path1    Resync is copying files to         - Path2
-INFO  : Resync updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful
-
-(04)  : test changed on one path - file1
-(05)  : touch-glob 2001-01-02 {datadir/} file5R.txt
-(06)  : touch-glob 2023-08-26 {datadir/} file7.txt
-(07)  : copy-as {datadir/}file5R.txt {path2/} file1.txt
-
-(08)  : test bisync with file changed during
-(09)  : concurrent-func
-(10)  : bisync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Building Path1 and Path2 listings
-INFO  : Path1 checking for diffs
-INFO  : Path2 checking for diffs
-INFO  : - Path2    File changed: size (larger), time (newer) - file1.txt
-INFO  : Path2:    1 changes:    0 new,    1 modified,    0 deleted
-INFO  : (Modified:    1 newer,    0 older,    1 larger,    0 smaller)
-INFO  : Applying changes
-INFO  : - Path2    Queue copy to Path1       - {path1/}file1.txt
-INFO  : - Path2    Do queued copies to                - Path1
-INFO  : Updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful
-
-(11)  : bisync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Building Path1 and Path2 listings
-INFO  : Path1 checking for diffs
-INFO  : Path2 checking for diffs
-INFO  : No changes found
-INFO  : Updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful

cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST

@@ -1 +0,0 @@
-This file is used for testing the health of rclone accesses to the local/remote file system.  Do not delete.

cmd/bisync/testdata/test_concurrent/modfiles/file1.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file10.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file11.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file2.txt

@@ -1 +0,0 @@
-Newer version

cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt

@@ -1 +0,0 @@
-This file is newer and not equal to 5R

cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt

@@ -1 +0,0 @@
-This file is newer and not equal to 5L

cmd/bisync/testdata/test_concurrent/modfiles/file6.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file7.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/scenario.txt

@@ -1,15 +0,0 @@
-test concurrent
-
-test initial bisync
-bisync resync
-
-test changed on one path - file1
-touch-glob 2001-01-02 {datadir/} file5R.txt
-touch-glob 2023-08-26 {datadir/} file7.txt
-copy-as {datadir/}file5R.txt {path2/} file1.txt
-
-test bisync with file changed during
-concurrent-func
-bisync
-
-bisync

cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log

@@ -80,7 +80,6 @@ INFO  : Path2 checking for diffs
 INFO  : Applying changes
 INFO  : - Path1    Queue copy to Path2       - {path2/}subdir
 INFO  : - Path1    Do queued copies to                - Path2
-INFO  : subdir: Making directory
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_resync/golden/test.log

@@ -23,7 +23,7 @@ INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful
 (05)  : move-listings empty-path1
 
-(06)  : test 2. resync with empty path2, resulting in syncing all content to path2.
+(06)  : test 2. resync with empty path2, resulting in synching all content to path2.
 (07)  : purge-children {path2/}
 (08)  : bisync resync
 INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.

cmd/bisync/testdata/test_resync/scenario.txt

@@ -1,6 +1,6 @@
 test resync
 # 1. Resync with empty Path1, resulting in copying all content FROM Path2
-# 2. Resync with empty Path2, resulting in syncing all content TO Path2
+# 2. Resync with empty Path2, resulting in synching all content TO Path2
 # 3. Exercise all of the various file difference scenarios during a resync:
 #    File       Path1       Path2           Expected action     Who wins
 #  - file1.txt  Exists      Missing         Sync Path1 >Path2   Path1
@@ -17,7 +17,7 @@ purge-children {path1/}
 bisync resync
 move-listings empty-path1
 
-test 2. resync with empty path2, resulting in syncing all content to path2.
+test 2. resync with empty path2, resulting in synching all content to path2.
 purge-children {path2/}
 bisync resync
 move-listings empty-path2

cmd/cmd.go

@@ -429,12 +429,11 @@ func initConfig() {
 		fs.Fatalf(nil, "Failed to start remote control: %v", err)
 	}
 
-	// Start the metrics server if configured and not running the "rc" command
-	if len(os.Args) >= 2 && os.Args[1] != "rc" {
+	// Start the metrics server if configured
 	_, err = rcserver.MetricsStart(ctx, &rc.Opt)
 	if err != nil {
 		fs.Fatalf(nil, "Failed to start metrics server: %v", err)
-		}
+
 	}
 
 	// Setup CPU profiling if desired

cmd/cmount/fs.go

@@ -121,6 +121,19 @@ func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *vfs.Dir, err
 	return leaf, dir, errc
 }
 
+// lookup a File given a path
+func (fsys *FS) lookupFile(path string) (file *vfs.File, errc int) {
+	node, errc := fsys.lookupNode(path)
+	if errc != 0 {
+		return nil, errc
+	}
+	file, ok := node.(*vfs.File)
+	if !ok {
+		return nil, -fuse.EISDIR
+	}
+	return file, 0
+}
+
 // get a node and handle from the path or from the fh if not fhUnset
 //
 // handle may be nil
@@ -141,9 +154,15 @@ func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
 	Size := uint64(node.Size())
 	Blocks := (Size + 511) / 512
 	modTime := node.ModTime()
+	Mode := node.Mode().Perm()
+	if node.IsDir() {
+		Mode |= fuse.S_IFDIR
+	} else {
+		Mode |= fuse.S_IFREG
+	}
 	//stat.Dev = 1
 	stat.Ino = node.Inode() // FIXME do we need to set the inode number?
-	stat.Mode = getMode(node)
+	stat.Mode = uint32(Mode)
 	stat.Nlink = 1
 	stat.Uid = fsys.VFS.Opt.UID
 	stat.Gid = fsys.VFS.Opt.GID
@@ -490,15 +509,14 @@ func (fsys *FS) Link(oldpath string, newpath string) (errc int) {
 
 // Symlink creates a symbolic link.
 func (fsys *FS) Symlink(target string, newpath string) (errc int) {
-	defer log.Trace(target, "newpath=%q, target=%q", newpath, target)("errc=%d", &errc)
-	return translateError(fsys.VFS.Symlink(target, newpath))
+	defer log.Trace(target, "newpath=%q", newpath)("errc=%d", &errc)
+	return -fuse.ENOSYS
 }
 
 // Readlink reads the target of a symbolic link.
 func (fsys *FS) Readlink(path string) (errc int, linkPath string) {
-	defer log.Trace(path, "")("errc=%v, linkPath=%q", &errc, linkPath)
-	linkPath, err := fsys.VFS.Readlink(path)
-	return translateError(err), linkPath
+	defer log.Trace(path, "")("linkPath=%q, errc=%d", &linkPath, &errc)
+	return -fuse.ENOSYS, ""
 }
 
 // Chmod changes the permission bits of a file.
@@ -562,7 +580,7 @@ func (fsys *FS) Getpath(path string, fh uint64) (errc int, normalisedPath string
 		return errc, ""
 	}
 	normalisedPath = node.Path()
-	if !strings.HasPrefix(normalisedPath, "/") {
+	if !strings.HasPrefix("/", normalisedPath) {
 		normalisedPath = "/" + normalisedPath
 	}
 	return 0, normalisedPath
@@ -597,8 +615,6 @@ func translateError(err error) (errc int) {
 		return -fuse.ENOSYS
 	case vfs.EINVAL:
 		return -fuse.EINVAL
-	case vfs.ELOOP:
-		return -fuse.ELOOP
 	}
 	fs.Errorf(nil, "IO error: %v", err)
 	return -fuse.EIO
@@ -630,22 +646,6 @@ func translateOpenFlags(inFlags int) (outFlags int) {
 	return outFlags
 }
 
-// get the Mode from a vfs Node
-func getMode(node os.FileInfo) uint32 {
-	vfsMode := node.Mode()
-	Mode := vfsMode.Perm()
-	if vfsMode&os.ModeDir != 0 {
-		Mode |= fuse.S_IFDIR
-	} else if vfsMode&os.ModeSymlink != 0 {
-		Mode |= fuse.S_IFLNK
-	} else if vfsMode&os.ModeNamedPipe != 0 {
-		Mode |= fuse.S_IFIFO
-	} else {
-		Mode |= fuse.S_IFREG
-	}
-	return uint32(Mode)
-}
-
 // Make sure interfaces are satisfied
 var (
 	_ fuse.FileSystemInterface = (*FS)(nil)

cmd/cmount/mount.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/rclone/rclone/cmd/mountlib"
@@ -34,6 +35,19 @@ func init() {
 	buildinfo.Tags = append(buildinfo.Tags, "cmount")
 }
 
+// Find the option string in the current options
+func findOption(name string, options []string) (found bool) {
+	for _, option := range options {
+		if option == "-o" {
+			continue
+		}
+		if strings.Contains(option, name) {
+			return true
+		}
+	}
+	return false
+}
+
 // mountOptions configures the options from the command line flags
 func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
 	// Options
@@ -79,9 +93,9 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.
 		if VFS.Opt.ReadOnly {
 			options = append(options, "-o", "ro")
 		}
-		//if opt.WritebackCache {
+		if opt.WritebackCache {
 			// FIXME? options = append(options, "-o", WritebackCache())
-		//}
+		}
 		if runtime.GOOS == "darwin" {
 			if opt.VolumeName != "" {
 				options = append(options, "-o", "volname="+opt.VolumeName)
@@ -97,7 +111,9 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.
 	for _, option := range opt.ExtraOptions {
 		options = append(options, "-o", option)
 	}
-	options = append(options, opt.ExtraFlags...)
+	for _, option := range opt.ExtraFlags {
+		options = append(options, option)
+	}
 	return options
 }
 

cmd/config/config.go

@@ -549,12 +549,12 @@ password to re-encrypt the config.
 
 When |--password-command| is called to change the password then the
 environment variable |RCLONE_PASSWORD_CHANGE=1| will be set. So if
-changing passwords programmatically you can use the environment
+changing passwords programatically you can use the environment
 variable to distinguish which password you must supply.
 
 Alternatively you can remove the password first (with |rclone config
 encryption remove|), then set it again with this command which may be
-easier if you don't mind the unencrypted config file being on the disk
+easier if you don't mind the unecrypted config file being on the disk
 briefly.
 `, "|", "`"),
 	RunE: func(command *cobra.Command, args []string) error {

cmd/copyto/copyto.go

@@ -43,8 +43,6 @@ This doesn't transfer files that are identical on src and dst, testing
 by size and modification time or MD5SUM.  It doesn't delete files from
 the destination.
 
-*If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'*
-
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
 `,
 	Annotations: map[string]string{

cmd/copyurl/copyurl.go

@@ -43,7 +43,7 @@ Setting |--auto-filename| will attempt to automatically determine the
 filename from the URL (after any redirections) and used in the
 destination path.
 
-With |--header-filename| in addition, if a specific filename is
+With |--auto-filename-header| in addition, if a specific filename is
 set in HTTP headers, it will be used instead of the name from the URL.
 With |--print-filename| in addition, the resulting file name will be
 printed.
@@ -54,7 +54,7 @@ destination if there is one with the same name.
 Setting |--stdout| or making the output file name |-|
 will cause the output to be written to standard output.
 
-### Troubleshooting
+### Troublshooting
 
 If you can't get |rclone copyurl| to work then here are some things you can try:
 

cmd/mount/dir.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
 	"syscall"
 	"time"
 
@@ -34,7 +33,7 @@ func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
 	a.Valid = time.Duration(d.fsys.opt.AttrTimeout)
 	a.Gid = d.VFS().Opt.GID
 	a.Uid = d.VFS().Opt.UID
-	a.Mode = d.Mode()
+	a.Mode = os.ModeDir | os.FileMode(d.VFS().Opt.DirPerms)
 	modTime := d.ModTime()
 	a.Atime = modTime
 	a.Mtime = modTime
@@ -141,13 +140,11 @@ var _ fusefs.NodeCreater = (*Dir)(nil)
 // Create makes a new file
 func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (node fusefs.Node, handle fusefs.Handle, err error) {
 	defer log.Trace(d, "name=%q", req.Name)("node=%v, handle=%v, err=%v", &node, &handle, &err)
-	// translate the fuse flags to os flags
-	osFlags := int(req.Flags) | os.O_CREATE
-	file, err := d.Dir.Create(req.Name, osFlags)
+	file, err := d.Dir.Create(req.Name, int(req.Flags))
 	if err != nil {
 		return nil, nil, translateError(err)
 	}
-	fh, err := file.Open(osFlags)
+	fh, err := file.Open(int(req.Flags) | os.O_CREATE)
 	if err != nil {
 		return nil, nil, translateError(err)
 	}
@@ -203,6 +200,7 @@ func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs
 	if !ok {
 		return fmt.Errorf("unknown Dir type %T", newDir)
 	}
+
 	err = d.Dir.Rename(req.OldName, req.NewName, destDir.Dir)
 	if err != nil {
 		return translateError(err)
@@ -241,24 +239,6 @@ func (d *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fusefs.Node)
 	return nil, syscall.ENOSYS
 }
 
-var _ fusefs.NodeSymlinker = (*Dir)(nil)
-
-// Symlink create a symbolic link.
-func (d *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (node fusefs.Node, err error) {
-	defer log.Trace(d, "newname=%v, target=%v", req.NewName, req.Target)("node=%v, err=%v", &node, &err)
-
-	newName := path.Join(d.Path(), req.NewName)
-	target := req.Target
-
-	n, err := d.VFS().CreateSymlink(target, newName)
-	if err != nil {
-		return nil, err
-	}
-
-	node = &File{n.(*vfs.File), d.fsys}
-	return node, nil
-}
-
 // Check interface satisfied
 var _ fusefs.NodeMknoder = (*Dir)(nil)
 

cmd/mount/file.go

@@ -32,7 +32,7 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
 	Blocks := (Size + 511) / 512
 	a.Gid = f.VFS().Opt.GID
 	a.Uid = f.VFS().Opt.UID
-	a.Mode = f.File.Mode() &^ os.ModeAppend
+	a.Mode = os.FileMode(f.VFS().Opt.FilePerms)
 	a.Size = Size
 	a.Atime = modTime
 	a.Mtime = modTime
@@ -129,11 +129,3 @@ func (f *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) er
 }
 
 var _ fusefs.NodeRemovexattrer = (*File)(nil)
-
-var _ fusefs.NodeReadlinker = (*File)(nil)
-
-// Readlink read symbolic link target.
-func (f *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (ret string, err error) {
-	defer log.Trace(f, "")("ret=%v, err=%v", &ret, &err)
-	return f.VFS().Readlink(f.Path())
-}

cmd/mount/fs.go

@@ -100,8 +100,6 @@ func translateError(err error) error {
 		return syscall.ENOSYS
 	case vfs.EINVAL:
 		return fuse.Errno(syscall.EINVAL)
-	case vfs.ELOOP:
-		return fuse.Errno(syscall.ELOOP)
 	}
 	fs.Errorf(nil, "IO error: %v", err)
 	return err

cmd/mount2/fs.go

@@ -51,14 +51,9 @@ func (f *FS) SetDebug(debug bool) {
 
 // get the Mode from a vfs Node
 func getMode(node os.FileInfo) uint32 {
-	vfsMode := node.Mode()
-	Mode := vfsMode.Perm()
-	if vfsMode&os.ModeDir != 0 {
+	Mode := node.Mode().Perm()
+	if node.IsDir() {
 		Mode |= fuse.S_IFDIR
-	} else if vfsMode&os.ModeSymlink != 0 {
-		Mode |= fuse.S_IFLNK
-	} else if vfsMode&os.ModeNamedPipe != 0 {
-		Mode |= fuse.S_IFIFO
 	} else {
 		Mode |= fuse.S_IFREG
 	}
@@ -133,8 +128,6 @@ func translateError(err error) syscall.Errno {
 		return syscall.ENOSYS
 	case vfs.EINVAL:
 		return syscall.EINVAL
-	case vfs.ELOOP:
-		return syscall.ELOOP
 	}
 	fs.Errorf(nil, "IO error: %v", err)
 	return syscall.EIO

cmd/mount2/node.go

@@ -227,7 +227,7 @@ type dirStream struct {
 // HasNext indicates if there are further entries. HasNext
 // might be called on already closed streams.
 func (ds *dirStream) HasNext() bool {
-	return ds.i < len(ds.nodes)+2
+	return ds.i < len(ds.nodes)
 }
 
 // Next retrieves the next entry. It is only called if HasNext
@@ -235,22 +235,7 @@ func (ds *dirStream) HasNext() bool {
 // indicate I/O errors
 func (ds *dirStream) Next() (de fuse.DirEntry, errno syscall.Errno) {
 	// defer log.Trace(nil, "")("de=%+v, errno=%v", &de, &errno)
-	if ds.i == 0 {
-		ds.i++
-		return fuse.DirEntry{
-			Mode: fuse.S_IFDIR,
-			Name: ".",
-			Ino:  0, // FIXME
-		}, 0
-	} else if ds.i == 1 {
-		ds.i++
-		return fuse.DirEntry{
-			Mode: fuse.S_IFDIR,
-			Name: "..",
-			Ino:  0, // FIXME
-		}, 0
-	}
-	fi := ds.nodes[ds.i-2]
+	fi := ds.nodes[ds.i]
 	de = fuse.DirEntry{
 		// Mode is the file's mode. Only the high bits (e.g. S_IFDIR)
 		// are considered.
@@ -458,31 +443,3 @@ func (n *Node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errn
 }
 
 var _ fusefs.NodeListxattrer = (*Node)(nil)
-
-var _ fusefs.NodeReadlinker = (*Node)(nil)
-
-// Readlink read symbolic link target.
-func (n *Node) Readlink(ctx context.Context) (ret []byte, err syscall.Errno) {
-	defer log.Trace(n, "")("ret=%v, err=%v", &ret, &err)
-	path := n.node.Path()
-	s, serr := n.node.VFS().Readlink(path)
-	return []byte(s), translateError(serr)
-}
-
-var _ fusefs.NodeSymlinker = (*Node)(nil)
-
-// Symlink create symbolic link.
-func (n *Node) Symlink(ctx context.Context, target, name string, out *fuse.EntryOut) (node *fusefs.Inode, err syscall.Errno) {
-	defer log.Trace(n, "name=%v, target=%v", name, target)("node=%v, err=%v", &node, &err)
-	fullPath := path.Join(n.node.Path(), name)
-	vfsNode, serr := n.node.VFS().CreateSymlink(target, fullPath)
-	if serr != nil {
-		return nil, translateError(serr)
-	}
-
-	n.fsys.setEntryOut(vfsNode, out)
-	newNode := newNode(n.fsys, vfsNode)
-	newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode})
-
-	return newInode, 0
-}

cmd/nfsmount/nfsmount_test.go

@@ -3,9 +3,6 @@
 package nfsmount
 
 import (
-	"context"
-	"errors"
-	"os"
 	"os/exec"
 	"runtime"
 	"testing"
@@ -33,24 +30,7 @@ func TestMount(t *testing.T) {
 		}
 		sudo = true
 	}
-	for _, cacheType := range []string{"memory", "disk", "symlink"} {
-		t.Run(cacheType, func(t *testing.T) {
 	nfs.Opt.HandleCacheDir = t.TempDir()
-			require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))
-			// Check we can create a handler
-			_, err := nfs.NewHandler(context.Background(), nil, &nfs.Opt)
-			if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {
-				t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")
-			}
-			require.NoError(t, err)
-			// Configure rclone via environment var since the mount gets run in a subprocess
-			_ = os.Setenv("RCLONE_NFS_CACHE_DIR", nfs.Opt.HandleCacheDir)
-			_ = os.Setenv("RCLONE_NFS_CACHE_TYPE", cacheType)
-			t.Cleanup(func() {
-				_ = os.Unsetenv("RCLONE_NFS_CACHE_DIR")
-				_ = os.Unsetenv("RCLONE_NFS_CACHE_TYPE")
-			})
+	require.NoError(t, nfs.Opt.HandleCache.Set("disk"))
 	vfstest.RunTests(t, false, vfscommon.CacheModeWrites, false, mount)
-		})
-	}
 }