lib/oauthutil: allow the browser opening function to be overridden

This patch ensures that we pass the context from the CreateBackend
call all the way to the creation of the oauth web server.

This means that when the command has finished the webserver will
definitely be shut down, and if the user abandons it (eg via an rc
call timing out or being cancelled) then it will be shut down too.

.github/workflows/build.yml

@@ -123,8 +123,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,77 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .
+            - name: Build and publish image
+              uses: docker/build-push-action@v6
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha, scope=${{ github.workflow }}
+                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -1,294 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*-
-
-name: Build & Push Docker Images
-
-# Trigger the workflow on push or pull request
-on:
-  push:
-    branches:
-      - '**'
-    tags:
-      - '**'
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        default: true
-
-jobs:
-  build-image:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runs-on: ubuntu-24.04
-          - platform: linux/386
-            runs-on: ubuntu-24.04
-          - platform: linux/arm64
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v7
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v6
-            runs-on: ubuntu-24.04-arm
-
-    name: Build Docker Image for ${{ matrix.platform }}
-    runs-on: ${{ matrix.runs-on }}
-
-    steps:
-      - name: Free Space
-        shell: bash
-        run: |
-          df -h .
-          # Remove android SDK
-          sudo rm -rf /usr/local/lib/android || true
-          # Remove .net runtime
-          sudo rm -rf /usr/share/dotnet || true
-          df -h .
-
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Set PLATFORM Variable
-        run: |
-          platform=${{ matrix.platform }}
-          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Set CACHE_NAME Variable
-        shell: python
-        run: |
-          import os, re
-
-          def slugify(input_string, max_length=63):
-            slug = input_string.lower()
-            slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
-            slug = slug.strip()
-            slug = re.sub(r'\s+', '-', slug)
-            slug = re.sub(r'-+', '-', slug)
-            slug = slug[:max_length]
-            slug = re.sub(r'[-]+$', '', slug)
-            return slug
-
-          ref_name_slug = "cache"
-
-          if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
-            ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"CACHE_NAME={ref_name_slug}\n")
-
-      - name: Get ImageOS
-        # There's no way around this, because "ImageOS" is only available to
-        # processes, but the setup-go action uses it in its key.
-        id: imageos
-        uses: actions/github-script@v7
-        with:
-          result-encoding: string
-          script: |
-            return process.env.ImageOS
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
-        with:
-          images: |
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Load Go Build Cache for Docker
-        id: go-cache
-        uses: actions/cache@v4
-        with:
-          key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          # Cache only the go builds, the module download is cached via the docker layer caching
-          path: |
-            go-build-cache
-
-      - name: Inject Go Build Cache into Docker
-        uses: reproducible-containers/buildkit-cache-dance@v3
-        with:
-          cache-map: |
-            {
-              "go-build-cache": "/root/.cache/go-build"
-            }
-          skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and Publish Image Digest
-        id: build
-        uses: docker/build-push-action@v6
-        with:
-          file: Dockerfile
-          context: .
-          provenance: false
-          # don't specify 'tags' here (error "get can't push tagged ref by digest")
-          # tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          annotations: ${{ steps.meta.outputs.annotations }}
-          platforms: ${{ matrix.platform }}
-          outputs: |
-            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          cache-to: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
-
-      - name: Export Image Digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload Image Digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM }}
-          path: /tmp/digests/*
-          retention-days: 1
-          if-no-files-found: error
-
-  merge-image:
-    name: Merge & Push Final Docker Image
-    runs-on: ubuntu-24.04
-    needs:
-      - build-image
-
-    steps:
-      - name: Download Image Digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digests-*
-          merge-multiple: true
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
-        with:
-          images: |
-            ${{ env.REPO_NAME }}
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Extract Tags
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
-          tags_string = " ".join(tags)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"TAGS={tags_string}\n")
-
-      - name: Extract Annotations
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
-          annotations_string = " ".join(annotations)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"ANNOTATIONS={annotations_string}\n")
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create & Push Manifest List
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create \
-          ${{ env.TAGS }} \
-          ${{ env.ANNOTATIONS }} \
-          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
-
-      - name: Inspect and Run Multi-Platform Image
-        run: |
-          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_docker_plugin.yml

@@ -1,45 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*-
-
-name: Release Build for Docker Plugin
-
-on:
-  release:
-    types: [published]
-
-jobs:
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,89 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKER_HUB_USER }}
+                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+            - name: Build and publish image
+              uses: docker/build-push-action@v6
+              with:
+                file: Dockerfile
+                context: .
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                push: true
+                tags: |
+                  rclone/rclone:latest
+                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
+                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
+                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+
+    build_docker_volume_plugin:
+        if: github.repository == 'rclone/rclone'
+        needs: build
+        runs-on: ubuntu-latest
+        name: Build docker plugin job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Build and publish docker plugin
+              shell: bash
+              run: |
+                VER=${GITHUB_REF#refs/tags/}
+                PLUGIN_USER=rclone
+                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+                    export PLUGIN_USER PLUGIN_ARCH
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+                done
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

Dockerfile

@@ -1,47 +1,19 @@
 FROM golang:alpine AS builder
 
-ARG CGO_ENABLED=0
-
+COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN echo "**** Set Go Environment Variables ****" && \
-    go env -w GOCACHE=/root/.cache/go-build
-
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        make \
-        bash \
-        gawk \
-        git
-
-COPY go.mod .
-COPY go.sum .
-
-RUN echo "**** Download Go Dependencies ****" && \
-    go mod download -x
-
-RUN echo "**** Verify Go Dependencies ****" && \
-    go mod verify
-
-COPY . .
-
-RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
-    echo "**** Build Binary ****" && \
-    make
-
-RUN echo "**** Print Version Binary ****" && \
-    ./rclone version
+RUN apk add --no-cache make bash gawk git
+RUN \
+  CGO_ENABLED=0 \
+  make
+RUN ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        ca-certificates \
-        fuse3 \
-        tzdata && \
-    echo "Enable user_allow_other in fuse" && \
-    echo "user_allow_other" >> /etc/fuse.conf
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
+  echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

RELEASE.md

@@ -86,16 +86,6 @@ build.
 Once it compiles locally, push it on a test branch and commit fixes
 until the tests pass.
 
-### Major versions
-
-The above procedure will not upgrade major versions, so v2 to v3.
-However this tool can show which major versions might need to be
-upgraded:
-
-    go run github.com/icholy/gomajor@latest list -major
-
-Expect API breakage when updating major versions.
-
 ## Tidy beta
 
 At some point after the release run

VERSION

@@ -1 +1 @@
-v1.69.1
+v1.69.0

backend/all/all.go

@@ -10,7 +10,6 @@ import (
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/cloudinary"
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"

backend/azureblob/azureblob.go

@@ -2162,9 +2162,6 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/azurefiles/azurefiles.go

@@ -393,10 +393,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 	policyClientOptions := policy.ClientOptions{
 		Transport: newTransporter(ctx),
 	}
-	backup := service.ShareTokenIntentBackup
 	clientOpt := service.ClientOptions{
-		ClientOptions:     policyClientOptions,
-		FileRequestIntent: &backup,
+		ClientOptions: policyClientOptions,
 	}
 
 	// Here we auth by setting one of cred, sharedKeyCred or f.client
@@ -899,7 +897,7 @@ func (o *Object) getMetadata(ctx context.Context) error {
 
 // Hash returns the MD5 of an object returning a lowercase hex string
 //
-// May make a network request because the [fs.List] method does not
+// May make a network request becaue the [fs.List] method does not
 // return MD5 hashes for DirEntry
 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
 	if ty != hash.MD5 {

backend/b2/api/types.go

@@ -42,10 +42,9 @@ type Bucket struct {
 
 // LifecycleRule is a single lifecycle rule
 type LifecycleRule struct {
-	DaysFromHidingToDeleting                        *int   `json:"daysFromHidingToDeleting"`
-	DaysFromUploadingToHiding                       *int   `json:"daysFromUploadingToHiding"`
-	DaysFromStartingToCancelingUnfinishedLargeFiles *int   `json:"daysFromStartingToCancelingUnfinishedLargeFiles"`
-	FileNamePrefix                                  string `json:"fileNamePrefix"`
+	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
+	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
+	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base

backend/b2/b2.go

@@ -299,13 +299,14 @@ type Fs struct {
 
 // Object describes a b2 object
 type Object struct {
-	fs       *Fs       // what this object is part of
-	remote   string    // The remote path
-	id       string    // b2 id of the file
-	modTime  time.Time // The modified time of the object if known
-	sha1     string    // SHA-1 hash if known
-	size     int64     // Size of the object
-	mimeType string    // Content-Type of the object
+	fs       *Fs               // what this object is part of
+	remote   string            // The remote path
+	id       string            // b2 id of the file
+	modTime  time.Time         // The modified time of the object if known
+	sha1     string            // SHA-1 hash if known
+	size     int64             // Size of the object
+	mimeType string            // Content-Type of the object
+	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
 }
 
 // ------------------------------------------------------------
@@ -1597,6 +1598,9 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if err != nil {
 		return err
 	}
+	// For now, just set "mtime" in metadata
+	o.meta = make(map[string]string, 1)
+	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
 	return nil
 }
 
@@ -1876,6 +1880,13 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		Info:            Info,
 	}
 
+	// Embryonic metadata support - just mtime
+	o.meta = make(map[string]string, 1)
+	modTime, err := parseTimeStringHelper(info.Info[timeKey])
+	if err == nil {
+		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
+	}
+
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -2220,7 +2231,6 @@ This will dump something like this showing the lifecycle rules.
         {
             "daysFromHidingToDeleting": 1,
             "daysFromUploadingToHiding": null,
-            "daysFromStartingToCancelingUnfinishedLargeFiles": null,
             "fileNamePrefix": ""
         }
     ]
@@ -2247,9 +2257,8 @@ overwrites will still cause versions to be made.
 See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 `,
 	Opts: map[string]string{
-		"daysFromHidingToDeleting":                        "After a file has been hidden for this many days it is deleted. 0 is off.",
-		"daysFromUploadingToHiding":                       "This many days after uploading a file is hidden",
-		"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
+		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
+		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
 	},
 }
 
@@ -2269,13 +2278,6 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 		}
 		newRule.DaysFromUploadingToHiding = &days
 	}
-	if daysStr := opt["daysFromStartingToCancelingUnfinishedLargeFiles"]; daysStr != "" {
-		days, err := strconv.Atoi(daysStr)
-		if err != nil {
-			return nil, fmt.Errorf("bad daysFromStartingToCancelingUnfinishedLargeFiles: %w", err)
-		}
-		newRule.DaysFromStartingToCancelingUnfinishedLargeFiles = &days
-	}
 	bucketName, _ := f.split("")
 	if bucketName == "" {
 		return nil, errors.New("bucket required")
@@ -2283,7 +2285,7 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 	}
 
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err

backend/b2/b2_internal_test.go

@@ -256,6 +256,12 @@ func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string
 			assert.Equal(t, v, got, k)
 		}
 
+		// mtime
+		for k, v := range metadata {
+			got := o.meta[k]
+			assert.Equal(t, v, got, k)
+		}
+
 		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
 
 		// Modification time from the x-bz-info-src_last_modified_millis header

backend/box/box.go

@@ -46,6 +46,7 @@ import (
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -64,10 +65,12 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		Scopes:       nil,
-		AuthURL:      "https://app.box.com/api/oauth2/authorize",
-		TokenURL:     "https://app.box.com/api/oauth2/token",
+	oauthConfig = &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://app.box.com/api/oauth2/authorize",
+			TokenURL: "https://app.box.com/api/oauth2/token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -255,9 +258,6 @@ func getQueryParams(boxConfig *api.ConfigJSON) map[string]string {
 
 func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) {
 	block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey))
-	if block == nil {
-		return nil, errors.New("box: failed to PEM decode private key")
-	}
 	if len(rest) > 0 {
 		return nil, fmt.Errorf("box: extra data included in private key: %w", err)
 	}

backend/chunker/chunker.go

@@ -2480,7 +2480,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	if len(data) > maxMetadataSizeWritten {
 		return nil, false, ErrMetaTooBig
 	}
-	if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
+	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
 		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON

backend/cloudinary/api/types.go

@@ -1,48 +0,0 @@
-// Package api has type definitions for cloudinary
-package api
-
-import (
-	"fmt"
-)
-
-// CloudinaryEncoder extends the built-in encoder
-type CloudinaryEncoder interface {
-	// FromStandardPath takes a / separated path in Standard encoding
-	// and converts it to a / separated path in this encoding.
-	FromStandardPath(string) string
-	// FromStandardName takes name in Standard encoding and converts
-	// it in this encoding.
-	FromStandardName(string) string
-	// ToStandardPath takes a / separated path in this encoding
-	// and converts it to a / separated path in Standard encoding.
-	ToStandardPath(string) string
-	// ToStandardName takes name in this encoding and converts
-	// it in Standard encoding.
-	ToStandardName(string) string
-	// Encoded root of the remote (as passed into NewFs)
-	FromStandardFullPath(string) string
-}
-
-// UpdateOptions was created to pass options from Update to Put
-type UpdateOptions struct {
-	PublicID     string
-	ResourceType string
-	DeliveryType string
-	AssetFolder  string
-	DisplayName  string
-}
-
-// Header formats the option as a string
-func (o *UpdateOptions) Header() (string, string) {
-	return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}
-
-// Mandatory returns whether the option must be parsed or can be ignored
-func (o *UpdateOptions) Mandatory() bool {
-	return false
-}
-
-// String formats the option into human-readable form
-func (o *UpdateOptions) String() string {
-	return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}

backend/cloudinary/cloudinary.go

@@ -1,711 +0,0 @@
-// Package cloudinary provides an interface to the Cloudinary DAM
-package cloudinary
-
-import (
-	"context"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cloudinary/cloudinary-go/v2"
-	SDKApi "github.com/cloudinary/cloudinary-go/v2/api"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin/search"
-	"github.com/cloudinary/cloudinary-go/v2/api/uploader"
-	"github.com/rclone/rclone/backend/cloudinary/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-	"github.com/zeebo/blake3"
-)
-
-// Cloudinary shouldn't have a trailing dot if there is no path
-func cldPathDir(somePath string) string {
-	if somePath == "" || somePath == "." {
-		return somePath
-	}
-	dir := path.Dir(somePath)
-	if dir == "." {
-		return ""
-	}
-	return dir
-}
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "cloudinary",
-		Description: "Cloudinary",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:      "cloud_name",
-				Help:      "Cloudinary Environment Name",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_key",
-				Help:      "Cloudinary API Key",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_secret",
-				Help:      "Cloudinary API Secret",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name: "upload_prefix",
-				Help: "Specify the API endpoint for environments out of the US",
-			},
-			{
-				Name: "upload_preset",
-				Help: "Upload Preset to select asset manipulation on upload",
-			},
-			{
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
-					encoder.EncodeSlash |
-					encoder.EncodeLtGt |
-					encoder.EncodeDoubleQuote |
-					encoder.EncodeQuestion |
-					encoder.EncodeAsterisk |
-					encoder.EncodePipe |
-					encoder.EncodeHash |
-					encoder.EncodePercent |
-					encoder.EncodeBackSlash |
-					encoder.EncodeDel |
-					encoder.EncodeCtl |
-					encoder.EncodeRightSpace |
-					encoder.EncodeInvalidUtf8 |
-					encoder.EncodeDot),
-			},
-			{
-				Name:     "eventually_consistent_delay",
-				Default:  fs.Duration(0),
-				Advanced: true,
-				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",
-			},
-		},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	CloudName                 string               `config:"cloud_name"`
-	APIKey                    string               `config:"api_key"`
-	APISecret                 string               `config:"api_secret"`
-	UploadPrefix              string               `config:"upload_prefix"`
-	UploadPreset              string               `config:"upload_preset"`
-	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EventuallyConsistentDelay fs.Duration          `config:"eventually_consistent_delay"`
-}
-
-// Fs represents a remote cloudinary server
-type Fs struct {
-	name     string
-	root     string
-	opt      Options
-	features *fs.Features
-	pacer    *fs.Pacer
-	srv      *rest.Client           // For downloading assets via the Cloudinary CDN
-	cld      *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK
-	lastCRUD time.Time
-}
-
-// Object describes a cloudinary object
-type Object struct {
-	fs           *Fs
-	remote       string
-	size         int64
-	modTime      time.Time
-	url          string
-	md5sum       string
-	publicID     string
-	resourceType string
-	deliveryType string
-}
-
-// NewFs constructs an Fs from the path, bucket:path
-func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize the Cloudinary client
-	cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)
-	}
-	cld.Admin.Client = *fshttp.NewClient(ctx)
-	cld.Upload.Client = *fshttp.NewClient(ctx)
-	if opt.UploadPrefix != "" {
-		cld.Config.API.UploadPrefix = opt.UploadPrefix
-	}
-	client := fshttp.NewClient(ctx)
-	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		cld:   cld,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
-		srv:   rest.NewClient(client),
-	}
-
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-	}).Fill(ctx, f)
-
-	if root != "" {
-		// Check to see if the root actually an existing file
-		remote := path.Base(root)
-		f.root = cldPathDir(root)
-		_, err := f.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
-				// File doesn't exist so return the previous root
-				f.root = root
-				return f, nil
-			}
-			return nil, err
-		}
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-// ------------------------------------------------------------
-
-// FromStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")
-}
-
-// FromStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
-}
-
-// ToStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")
-}
-
-// ToStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
-}
-
-// FromStandardFullPath encodes a full path to Cloudinary standard
-func (f *Fs) FromStandardFullPath(dir string) string {
-	return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))
-}
-
-// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK
-func (f *Fs) ToAssetFolderAPI(dir string) string {
-	return strings.ReplaceAll(dir, "%", "%25")
-}
-
-// ToDisplayNameElastic encodes a special case of elasticsearch
-func (f *Fs) ToDisplayNameElastic(dir string) string {
-	return strings.ReplaceAll(dir, "!", "\\!")
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// WaitEventuallyConsistent waits till the FS is eventually consistent
-func (f *Fs) WaitEventuallyConsistent() {
-	if f.opt.EventuallyConsistentDelay == fs.Duration(0) {
-		return
-	}
-	delay := time.Duration(f.opt.EventuallyConsistentDelay)
-	timeSinceLastCRUD := time.Since(f.lastCRUD)
-	if timeSinceLastCRUD < delay {
-		time.Sleep(delay - timeSinceLastCRUD)
-	}
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("Cloudinary root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// List the objects and directories in dir into entries
-func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
-	remotePrefix := f.FromStandardFullPath(dir)
-	if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {
-		remotePrefix += "/"
-	}
-
-	var entries fs.DirEntries
-	dirs := make(map[string]struct{})
-	nextCursor := ""
-	f.WaitEventuallyConsistent()
-	for {
-		// user the folders api to list folders.
-		folderParams := admin.SubFoldersParams{
-			Folder:     f.ToAssetFolderAPI(remotePrefix),
-			MaxResults: 500,
-		}
-		if nextCursor != "" {
-			folderParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list sub-folders: %w", err)
-		}
-		if results.Error.Message != "" {
-			if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {
-				return nil, fs.ErrorDirNotFound
-			}
-
-			return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)
-		}
-
-		for _, folder := range results.Folders {
-			relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))
-			parts := strings.Split(relativePath, "/")
-
-			// It's a directory
-			dirName := parts[len(parts)-1]
-			if _, found := dirs[dirName]; !found {
-				d := fs.NewDir(path.Join(dir, dirName), time.Time{})
-				entries = append(entries, d)
-				dirs[dirName] = struct{}{}
-			}
-		}
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	for {
-		// Use the assets.AssetsByAssetFolder API to list assets
-		assetsParams := admin.AssetsByAssetFolderParams{
-			AssetFolder: remotePrefix,
-			MaxResults:  500,
-		}
-		if nextCursor != "" {
-			assetsParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list assets: %w", err)
-		}
-
-		for _, asset := range results.Assets {
-			remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
-			if dir != "" {
-				remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
-			}
-			o := &Object{
-				fs:           f,
-				remote:       remote,
-				size:         int64(asset.Bytes),
-				modTime:      asset.CreatedAt,
-				url:          asset.SecureURL,
-				publicID:     asset.PublicID,
-				resourceType: asset.AssetType,
-				deliveryType: asset.Type,
-			}
-			entries = append(entries, o)
-		}
-
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	return entries, nil
-}
-
-// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	searchParams := search.Query{
-		Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",
-			f.FromStandardFullPath(cldPathDir(remote)),
-			f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),
-		SortBy:     []search.SortByField{{"uploaded_at": "desc"}},
-		MaxResults: 2,
-	}
-	var results *admin.SearchResult
-	f.WaitEventuallyConsistent()
-	err := f.pacer.Call(func() (bool, error) {
-		var err1 error
-		results, err1 = f.cld.Admin.Search(ctx, searchParams)
-		if err1 == nil && results.TotalCount != len(results.Assets) {
-			err1 = errors.New("partial response so waiting for eventual consistency")
-		}
-		return shouldRetry(ctx, nil, err1)
-	})
-	if err != nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-	if results.TotalCount == 0 || len(results.Assets) == 0 {
-		return nil, fs.ErrorObjectNotFound
-	}
-	asset := results.Assets[0]
-
-	o := &Object{
-		fs:           f,
-		remote:       remote,
-		size:         int64(asset.Bytes),
-		modTime:      asset.UploadedAt,
-		url:          asset.SecureURL,
-		md5sum:       asset.Etag,
-		publicID:     asset.PublicID,
-		resourceType: asset.ResourceType,
-		deliveryType: asset.Type,
-	}
-
-	return o, nil
-}
-
-func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {
-	payload := []byte(path.Join(assetFolder, displayName))
-	hash := blake3.Sum256(payload)
-	return hex.EncodeToString(hash[:])
-}
-
-// Put uploads content to Cloudinary
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if src.Size() == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	params := uploader.UploadParams{
-		UploadPreset: f.opt.UploadPreset,
-	}
-
-	updateObject := false
-	var modTime time.Time
-	for _, option := range options {
-		if updateOptions, ok := option.(*api.UpdateOptions); ok {
-			if updateOptions.PublicID != "" {
-				updateObject = true
-				params.Overwrite = SDKApi.Bool(true)
-				params.Invalidate = SDKApi.Bool(true)
-				params.PublicID = updateOptions.PublicID
-				params.ResourceType = updateOptions.ResourceType
-				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
-				params.AssetFolder = updateOptions.AssetFolder
-				params.DisplayName = updateOptions.DisplayName
-				modTime = src.ModTime(ctx)
-			}
-		}
-	}
-	if !updateObject {
-		params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))
-		params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))
-		// We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).
-		// We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.
-		// Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.
-		params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))
-	}
-	uploadResult, err := f.cld.Upload.Upload(ctx, in, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)
-	}
-	if !updateObject {
-		modTime = uploadResult.CreatedAt
-	}
-	if uploadResult.Error.Message != "" {
-		return nil, errors.New(uploadResult.Error.Message)
-	}
-
-	o := &Object{
-		fs:           f,
-		remote:       src.Remote(),
-		size:         int64(uploadResult.Bytes),
-		modTime:      modTime,
-		url:          uploadResult.SecureURL,
-		md5sum:       uploadResult.Etag,
-		publicID:     uploadResult.PublicID,
-		resourceType: uploadResult.ResourceType,
-		deliveryType: uploadResult.Type,
-	}
-	return o, nil
-}
-
-// Precision of the remote
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash sets
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// Mkdir creates empty folders
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}
-	res, err := f.cld.Admin.CreateFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// Rmdir deletes empty folders
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	// Additional test because Cloudinary will delete folders without
-	// assets, regardless of empty sub-folders
-	folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))
-	folderParams := admin.SubFoldersParams{
-		Folder:     folder,
-		MaxResults: 1,
-	}
-	results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-	if err != nil {
-		return err
-	}
-	if results.TotalCount > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-
-	params := admin.DeleteFolderParams{Folder: folder}
-	res, err := f.cld.Admin.DeleteFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {
-			return fs.ErrorDirNotFound
-		}
-
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	420, // Too Many Requests (legacy)
-	429, // Too Many Requests
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	if err != nil {
-		tryAgain := "Try again on "
-		if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
-			layout := "2006-01-02 15:04:05 UTC"
-			dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
-			timestamp, err2 := time.Parse(layout, dateStr)
-			if err2 == nil {
-				return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
-			}
-		}
-
-		fs.Debugf(nil, "Retrying API error %v", err)
-		return true, err
-	}
-
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-// ------------------------------------------------------------
-
-// Hash returns the MD5 of an object
-func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
-	if ty != hash.MD5 {
-		return "", hash.ErrUnsupported
-	}
-	return o.md5sum, nil
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the object
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// Size of object in bytes
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// Storable returns if this object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-// Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: o.url,
-		Options: options,
-	}
-	var offset int64
-	var count int64
-	var key string
-	var value string
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-			key, value = option.Header()
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size - offset
-			key, value = option.Header()
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if key != "" && value != "" {
-		opts.ExtraHeaders = make(map[string]string)
-		opts.ExtraHeaders[key] = value
-	}
-	// Make sure that the asset is fully available
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
-		if err == nil {
-			cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))
-			if clErr == nil && count == int64(cl) {
-				return false, nil
-			}
-		}
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)
-	}
-	return resp.Body, err
-}
-
-// Update the object with the contents of the io.Reader
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	options = append(options, &api.UpdateOptions{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		DeliveryType: o.deliveryType,
-		DisplayName:  api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),
-		AssetFolder:  o.fs.FromStandardFullPath(cldPathDir(o.Remote())),
-	})
-	updatedObj, err := o.fs.Put(ctx, in, src, options...)
-	if err != nil {
-		return err
-	}
-	if uo, ok := updatedObj.(*Object); ok {
-		o.size = uo.size
-		o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time
-		o.url = uo.url
-		o.md5sum = uo.md5sum
-		o.publicID = uo.publicID
-		o.resourceType = uo.resourceType
-		o.deliveryType = uo.deliveryType
-	}
-	return nil
-}
-
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	params := uploader.DestroyParams{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		Type:         o.deliveryType,
-	}
-	res, dErr := o.fs.cld.Upload.Destroy(ctx, params)
-	o.fs.lastCRUD = time.Now()
-	if dErr != nil {
-		return dErr
-	}
-
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	if res.Result != "ok" {
-		return errors.New(res.Result)
-	}
-
-	return nil
-}

backend/cloudinary/cloudinary_test.go

@@ -1,23 +0,0 @@
-// Test Cloudinary filesystem interface
-
-package cloudinary_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/cloudinary"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	name := "TestCloudinary"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      name + ":",
-		NilObject:       (*cloudinary.Object)(nil),
-		SkipInvalidUTF8: true,
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "eventually_consistent_delay", Value: "7"},
-		},
-	})
-}

backend/drive/drive.go

@@ -80,10 +80,9 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	driveConfig = &oauthutil.Config{
+	driveConfig = &oauth2.Config{
 		Scopes:       []string{scopePrefix + "drive"},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
+		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -203,6 +202,7 @@ func driveScopesContainsAppFolder(scopes []string) bool {
 		if scope == scopePrefix+"drive.appfolder" {
 			return true
 		}
+
 	}
 	return false
 }
@@ -1211,7 +1211,6 @@ func fixMimeType(mimeTypeIn string) string {
 	}
 	return mimeTypeOut
 }
-
 func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	out = make(map[string][]string, len(in))
 	for k, v := range in {
@@ -1222,11 +1221,9 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	}
 	return out
 }
-
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
 }
-
 func isLinkMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/x-link-")
 }
@@ -1659,8 +1656,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	ctx context.Context, remote string, info *drive.File,
-	extension, exportName, exportMimeType string, isDocument bool,
-) (o fs.Object, err error) {
+	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
@@ -1851,7 +1847,6 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
-
 func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
@@ -1897,8 +1892,7 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
 func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
-	extension, mimeType string, isDocument bool,
-) {
+	extension, mimeType string, isDocument bool) {
 	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
@@ -2694,7 +2688,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if shortcutID != "" {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
-	trashedFiles := false
+	var trashedFiles = false
 	if check {
 		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
 			if !item.Trashed {
@@ -2931,6 +2925,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
+
 	if err != nil {
 		return err
 	}
@@ -3191,7 +3186,6 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
-
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
@@ -3995,13 +3989,14 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	case "query":
 		if len(arg) == 1 {
 			query := arg[0]
-			results, err := f.query(ctx, query)
+			var results, err = f.query(ctx, query)
 			if err != nil {
 				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
 			}
 			return results, nil
+		} else {
+			return nil, errors.New("need a query argument")
 		}
-		return nil, errors.New("need a query argument")
 	case "rescue":
 		dirID := ""
 		_, delete := opt["delete"]
@@ -4061,7 +4056,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	return "", hash.ErrUnsupported
 }
-
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
 		return "", hash.ErrUnsupported
@@ -4076,8 +4070,7 @@ func (o *baseObject) Size() int64 {
 
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
-	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error,
-) {
+	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -4290,13 +4283,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	return o.baseObject.open(ctx, o.url, options...)
 }
-
 func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
 	var offset, end int64 = 0, -1
-	newOptions := options[:0]
+	var newOptions = options[:0]
 	for _, o := range options {
 		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
@@ -4323,10 +4315,9 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
 	}
 	return
 }
-
 func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
-	data := o.content
+	var data = o.content
 	for _, option := range options {
 		switch x := option.(type) {
 		case *fs.SeekOption:
@@ -4351,8 +4342,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
-	src fs.ObjectInfo,
-) (info *drive.File, err error) {
+	src fs.ObjectInfo) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
 	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
@@ -4430,7 +4420,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	return nil
 }
-
 func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	srcMimeType := fs.MimeType(ctx, src)
 	importMimeType := ""
@@ -4526,7 +4515,6 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
-
 func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }

backend/dropbox/batcher.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fs/fserrors"
 )
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
@@ -20,10 +21,14 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
 		}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/dropbox/dropbox.go

@@ -47,7 +47,6 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -94,7 +93,7 @@ const (
 
 var (
 	// Description of how to auth for this app
-	dropboxConfig = &oauthutil.Config{
+	dropboxConfig = &oauth2.Config{
 		Scopes: []string{
 			"files.metadata.write",
 			"files.content.write",
@@ -109,8 +108,7 @@ var (
 		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
 		// 	TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
 		// },
-		AuthURL:      dropbox.OAuthEndpoint("").AuthURL,
-		TokenURL:     dropbox.OAuthEndpoint("").TokenURL,
+		Endpoint:     dropbox.OAuthEndpoint(""),
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -135,7 +133,7 @@ var (
 )
 
 // Gets an oauth config with the right scopes
-func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
+func getOauthConfig(m configmap.Mapper) *oauth2.Config {
 	// If not impersonating, use standard scopes
 	if impersonate, _ := m.Get("impersonate"); impersonate == "" {
 		return dropboxConfig
@@ -318,46 +316,32 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Some specific errors which should be excluded from retries
-func shouldRetryExclude(ctx context.Context, err error) (bool, error) {
-	if err == nil {
-		return false, err
-	}
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	// First check for specific errors
-	//
-	// These come back from the SDK in a whole host of different
-	// error types, but there doesn't seem to be a consistent way
-	// of reading the error cause, so here we just check using the
-	// error string which isn't perfect but does the job.
+	if err == nil {
+		return false, err
+	}
 	errString := err.Error()
+	// First check for specific errors
 	if strings.Contains(errString, "insufficient_space") {
 		return false, fserrors.FatalError(err)
 	} else if strings.Contains(errString, "malformed_path") {
 		return false, fserrors.NoRetryError(err)
 	}
-	return true, err
-}
-
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if retry, err := shouldRetryExclude(ctx, err); !retry {
-		return retry, err
-	}
 	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
-			fs.Logf(nil, "Error %v. Too many requests or write operations. Trying again in %d seconds.", err, e.RateLimitError.RetryAfter)
+			fs.Logf(errString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
 			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
 	// Keep old behavior for backward compatibility
-	errString := err.Error()
 	if strings.Contains(errString, "too_many_write_operations") || strings.Contains(errString, "too_many_requests") || errString == "" {
 		return true, err
 	}
@@ -1036,20 +1020,13 @@ func (f *Fs) Precision() time.Duration {
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
 
-	// Find and remove existing object
-	cleanup, err := operations.RemoveExisting(ctx, f, remote, "server side copy")
-	if err != nil {
-		return nil, err
-	}
-	defer cleanup(&err)
-
 	// Temporary Object under construction
 	dstObj := &Object{
 		fs:     f,
@@ -1063,6 +1040,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
 		},
 	}
+	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.CopyV2(&arg)
@@ -1714,10 +1692,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
 		}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/gofile/gofile.go

@@ -1214,7 +1214,7 @@ func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
@@ -1228,19 +1228,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		return nil, fmt.Errorf("can't copy %q -> %q as are same name", srcPath, dstPath)
 	}
 
-	// Find existing object
-	existingObj, err := f.NewObject(ctx, remote)
-	if err == nil {
-		defer func() {
-			// Don't remove existing object if returning an error
-			if err != nil {
-				return
-			}
-			fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
-			err = existingObj.Remove(ctx)
-		}()
-	}
-
 	// Create temporary object
 	dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage.go

@@ -60,17 +60,14 @@ const (
 	minSleep                    = 10 * time.Millisecond
 )
 
-var (
-	// Description of how to auth for this app
-	storageConfig = &oauthutil.Config{
-		Scopes:       []string{storage.DevstorageReadWriteScope},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectURL,
-	}
-)
+// Description of how to auth for this app
+var storageConfig = &oauth2.Config{
+	Scopes:       []string{storage.DevstorageReadWriteScope},
+	Endpoint:     google.Endpoint,
+	ClientID:     rcloneClientID,
+	ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
+	RedirectURL:  oauthutil.RedirectURL,
+}
 
 // Register with Fs
 func init() {

backend/googlephotos/googlephotos.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 )
 
@@ -59,14 +60,13 @@ const (
 
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
+	oauthConfig = &oauth2.Config{
 		Scopes: []string{
 			"openid",
 			"profile",
 			scopeReadWrite, // this must be at position scopeAccess
 		},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
+		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -1168,7 +1168,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		errors := make([]error, 1)
 		results := make([]*api.MediaItem, 1)
 		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
-		if err == nil {
+		if err != nil {
 			err = errors[0]
 			info = results[0]
 		}

backend/googlephotos/googlephotos_test.go

@@ -2,7 +2,6 @@ package googlephotos
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,7 +35,7 @@ func TestIntegration(t *testing.T) {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
-	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
+	if err == fs.ErrorNotFoundInConfigFile {
 		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)

backend/hidrive/hidrive.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -47,9 +48,11 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app.
-	oauthConfig = &oauthutil.Config{
-		AuthURL:      "https://my.hidrive.com/client/authorize",
-		TokenURL:     "https://my.hidrive.com/oauth2/token",
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://my.hidrive.com/client/authorize",
+			TokenURL: "https://my.hidrive.com/oauth2/token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.TitleBarRedirectURL,

backend/http/http.go

@@ -331,13 +331,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
-	trimmedRemote := strings.TrimLeft(remote, "/") // remove leading "/" since we always have it in f.endpointURL
 	if f.opt.NoEscape {
 		// Directly concatenate without escaping, no_escape behavior
-		return f.endpointURL + trimmedRemote
+		return f.endpointURL + remote
 	}
 	// Default behavior
-	return f.endpointURL + rest.URLPathEscape(trimmedRemote)
+	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
 // Errors returned by parseName

backend/http/http_internal_test.go

@@ -191,33 +191,6 @@ func TestNewObject(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
-func TestNewObjectWithLeadingSlash(t *testing.T) {
-	f := prepare(t)
-
-	o, err := f.NewObject(context.Background(), "/four/under four.txt")
-	require.NoError(t, err)
-
-	assert.Equal(t, "/four/under four.txt", o.Remote())
-	assert.Equal(t, int64(8+lineEndSize), o.Size())
-	_, ok := o.(*Object)
-	assert.True(t, ok)
-
-	// Test the time is correct on the object
-
-	tObj := o.ModTime(context.Background())
-
-	fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
-	require.NoError(t, err)
-	tFile := fi.ModTime()
-
-	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
-
-	// check object not found
-	o, err = f.NewObject(context.Background(), "/not found.txt")
-	assert.Nil(t, o)
-	assert.Equal(t, fs.ErrorObjectNotFound, err)
-}
-
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

backend/iclouddrive/iclouddrive.go

@@ -445,7 +445,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// build request
-	// can't use normal rename as file needs to be "activated" first
+	// cant use normal rename as file needs to be "activated" first
 
 	r := api.NewUpdateFileInfo()
 	r.DocumentID = doc.DocumentID

backend/imagekit/client/media.go

@@ -75,7 +75,7 @@ type MoveFolderParam struct {
 	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
 }
 
-// JobIDResponse represents response struct with JobID for folder operations
+// JobIDResponse respresents response struct with JobID for folder operations
 type JobIDResponse struct {
 	JobID string `json:"jobId"`
 }

backend/jottacloud/jottacloud.go

@@ -277,9 +277,11 @@ machines.`)
 		m.Set(configClientID, teliaseCloudClientID)
 		m.Set(configTokenURL, teliaseCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     teliaseCloudAuthURL,
-				TokenURL:    teliaseCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  teliaseCloudAuthURL,
+					TokenURL: teliaseCloudTokenURL,
+				},
 				ClientID:    teliaseCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -290,9 +292,11 @@ machines.`)
 		m.Set(configClientID, telianoCloudClientID)
 		m.Set(configTokenURL, telianoCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     telianoCloudAuthURL,
-				TokenURL:    telianoCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  telianoCloudAuthURL,
+					TokenURL: telianoCloudTokenURL,
+				},
 				ClientID:    telianoCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -303,9 +307,11 @@ machines.`)
 		m.Set(configClientID, tele2CloudClientID)
 		m.Set(configTokenURL, tele2CloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     tele2CloudAuthURL,
-				TokenURL:    tele2CloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  tele2CloudAuthURL,
+					TokenURL: tele2CloudTokenURL,
+				},
 				ClientID:    tele2CloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -316,9 +322,11 @@ machines.`)
 		m.Set(configClientID, onlimeCloudClientID)
 		m.Set(configTokenURL, onlimeCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     onlimeCloudAuthURL,
-				TokenURL:    onlimeCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  onlimeCloudAuthURL,
+					TokenURL: onlimeCloudTokenURL,
+				},
 				ClientID:    onlimeCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -916,17 +924,19 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	}
 
 	baseClient := fshttp.NewClient(ctx)
-	oauthConfig := &oauthutil.Config{
-		AuthURL:  defaultTokenURL,
-		TokenURL: defaultTokenURL,
+	oauthConfig := &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
+		},
 	}
 	if ver == configVersion {
 		oauthConfig.ClientID = defaultClientID
 		// if custom endpoints are set use them else stick with defaults
 		if tokenURL, ok := m.Get(configTokenURL); ok {
-			oauthConfig.TokenURL = tokenURL
+			oauthConfig.Endpoint.TokenURL = tokenURL
 			// jottacloud is weird. we need to use the tokenURL as authURL
-			oauthConfig.AuthURL = tokenURL
+			oauthConfig.Endpoint.AuthURL = tokenURL
 		}
 	} else if ver == legacyConfigVersion {
 		clientID, ok := m.Get(configClientID)
@@ -940,8 +950,8 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 		oauthConfig.ClientID = clientID
 		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-		oauthConfig.TokenURL = legacyTokenURL
-		oauthConfig.AuthURL = legacyTokenURL
+		oauthConfig.Endpoint.TokenURL = legacyTokenURL
+		oauthConfig.Endpoint.AuthURL = legacyTokenURL
 
 		// add the request filter to fix token refresh
 		if do, ok := baseClient.Transport.(interface {

backend/local/about_windows.go

@@ -5,18 +5,18 @@ package local
 import (
 	"context"
 	"fmt"
+	"syscall"
 	"unsafe"
 
 	"github.com/rclone/rclone/fs"
-	"golang.org/x/sys/windows"
 )
 
-var getFreeDiskSpace = windows.NewLazySystemDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
+var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var available, total, free int64
-	root, e := windows.UTF16PtrFromString(f.root)
+	root, e := syscall.UTF16PtrFromString(f.root)
 	if e != nil {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e)
 	}
@@ -26,7 +26,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
 		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes
 	)
-	if e1 != windows.Errno(0) {
+	if e1 != syscall.Errno(0) {
 		return nil, fmt.Errorf("failed to read disk usage: %w", e1)
 	}
 	usage := &fs.Usage{

backend/local/lchmod.go

@@ -1,16 +0,0 @@
-//go:build windows || plan9 || js || linux
-
-package local
-
-import "os"
-
-const haveLChmod = false
-
-// lChmod changes the mode of the named file to mode. If the file is a symbolic
-// link, it changes the link, not the target. If there is an error,
-// it will be of type *PathError.
-func lChmod(name string, mode os.FileMode) error {
-	// Can't do this safely on this OS - chmoding a symlink always
-	// changes the destination.
-	return nil
-}

backend/local/lchmod_unix.go

@@ -1,41 +0,0 @@
-//go:build !windows && !plan9 && !js && !linux
-
-package local
-
-import (
-	"os"
-	"syscall"
-
-	"golang.org/x/sys/unix"
-)
-
-const haveLChmod = true
-
-// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
-//
-// Borrowed from the syscall source since it isn't public.
-func syscallMode(i os.FileMode) (o uint32) {
-	o |= uint32(i.Perm())
-	if i&os.ModeSetuid != 0 {
-		o |= syscall.S_ISUID
-	}
-	if i&os.ModeSetgid != 0 {
-		o |= syscall.S_ISGID
-	}
-	if i&os.ModeSticky != 0 {
-		o |= syscall.S_ISVTX
-	}
-	return o
-}
-
-// lChmod changes the mode of the named file to mode. If the file is a symbolic
-// link, it changes the link, not the target. If there is an error,
-// it will be of type *PathError.
-func lChmod(name string, mode os.FileMode) error {
-	// NB linux does not support AT_SYMLINK_NOFOLLOW as a parameter to fchmodat
-	// and returns ENOTSUP if you try, so we don't support this on linux
-	if e := unix.Fchmodat(unix.AT_FDCWD, name, syscallMode(mode), unix.AT_SYMLINK_NOFOLLOW); e != nil {
-		return &os.PathError{Op: "lChmod", Path: name, Err: e}
-	}
-	return nil
-}

backend/local/lchtimes.go

@@ -1,4 +1,4 @@
-//go:build plan9 || js
+//go:build windows || plan9 || js
 
 package local
 

backend/local/lchtimes_windows.go

@@ -1,19 +0,0 @@
-//go:build windows
-
-package local
-
-import (
-	"time"
-)
-
-const haveLChtimes = true
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	return setTimes(name, atime, mtime, time.Time{}, true)
-}

backend/local/local.go

@@ -34,6 +34,7 @@ import (
 // Constants
 const (
 	devUnset   = 0xdeadbeefcafebabe                                     // a device id meaning it is unset
+	linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
 	useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
 )
 
@@ -100,8 +101,10 @@ Metadata is supported on files and directories.
 			},
 			{
 				Name:     "links",
-				Help:     "Translate symlinks to/from regular files with a '" + fs.LinkSuffix + "' extension for the local backend.",
+				Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.",
 				Default:  false,
+				NoPrefix: true,
+				ShortOpt: "l",
 				Advanced: true,
 			},
 			{
@@ -376,22 +379,17 @@ type Directory struct {
 
 var (
 	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-	errLinksNeedsSuffix  = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
+	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
 )
 
 // NewFs constructs an Fs from the path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	ci := fs.GetConfig(ctx)
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
 	}
-	// Override --local-links with --links if set
-	if ci.Links {
-		opt.TranslateSymlinks = true
-	}
 	if opt.TranslateSymlinks && opt.FollowSymlinks {
 		return nil, errLinksAndCopyLinks
 	}
@@ -437,9 +435,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
 	// Check to see if this is a .rclonelink if not found
-	hasLinkSuffix := strings.HasSuffix(f.root, fs.LinkSuffix)
+	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
 	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
-		fi, err = f.lstat(strings.TrimSuffix(f.root, fs.LinkSuffix))
+		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
 	}
 	if err == nil && f.isRegular(fi.Mode()) {
 		// Handle the odd case, that a symlink was specified by name without the link suffix
@@ -510,8 +508,8 @@ func (f *Fs) caseInsensitive() bool {
 //
 // for regular files, localPath is returned unchanged
 func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
-	newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
+	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
+	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
 	return newLocalPath, isTranslatedLink
 }
 
@@ -694,7 +692,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			} else {
 				// Check whether this link should be translated
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += fs.LinkSuffix
+					newRemote += linkSuffix
 				}
 				// Don't include non directory if not included
 				// we leave directory filtering to the layer above

backend/local/local_internal_test.go

@@ -110,7 +110,7 @@ func TestSymlink(t *testing.T) {
 	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
 
 	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+fs.LinkSuffix, "file.txt", modTime2)
+	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
 
 	// Object viewed as destination
 	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
@@ -139,7 +139,7 @@ func TestSymlink(t *testing.T) {
 
 	// Create a symlink
 	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+fs.LinkSuffix, "file.txt", modTime3, false)
+	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
 	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
 	if haveLChtimes {
 		r.CheckLocalItems(t, file1, file2, file3)
@@ -155,9 +155,9 @@ func TestSymlink(t *testing.T) {
 	assert.Equal(t, "file.txt", linkText)
 
 	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+fs.LinkSuffix)
+	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+linkSuffix)
 	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+fs.LinkSuffix, o.Remote())
+	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
 	assert.Equal(t, int64(8), o.Size())
 
 	// Check that NewObject doesn't see the non suffixed version
@@ -165,7 +165,7 @@ func TestSymlink(t *testing.T) {
 	require.Equal(t, fs.ErrorObjectNotFound, err)
 
 	// Check that NewFs works with the suffixed version and --links
-	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+fs.LinkSuffix), configmap.Simple{
+	f2, err := NewFs(ctx, "local", filepath.Join(dir, "symlink2.txt"+linkSuffix), configmap.Simple{
 		"links": "true",
 	})
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -268,66 +268,22 @@ func TestMetadata(t *testing.T) {
 	r := fstest.NewRun(t)
 	const filePath = "metafile.txt"
 	when := time.Now()
+	const dayLength = len("2001-01-01")
+	whenRFC := when.Format(time.RFC3339Nano)
 	r.WriteFile(filePath, "metadata file contents", when)
 	f := r.Flocal.(*Fs)
 
-	// Set fs into "-l" / "--links" mode
-	f.opt.TranslateSymlinks = true
-
-	// Write a symlink to the file
-	symlinkPath := "metafile-link.txt"
-	osSymlinkPath := filepath.Join(f.root, symlinkPath)
-	symlinkPath += fs.LinkSuffix
-	require.NoError(t, os.Symlink(filePath, osSymlinkPath))
-	symlinkModTime := fstest.Time("2002-02-03T04:05:10.123123123Z")
-	require.NoError(t, lChtimes(osSymlinkPath, symlinkModTime, symlinkModTime))
-
 	// Get the object
 	obj, err := f.NewObject(ctx, filePath)
 	require.NoError(t, err)
 	o := obj.(*Object)
 
-	// Get the symlink object
-	symlinkObj, err := f.NewObject(ctx, symlinkPath)
-	require.NoError(t, err)
-	symlinkO := symlinkObj.(*Object)
-
-	// Record metadata for o
-	oMeta, err := o.Metadata(ctx)
-	require.NoError(t, err)
-
-	// Test symlink first to check it doesn't mess up file
-	t.Run("Symlink", func(t *testing.T) {
-		testMetadata(t, r, symlinkO, symlinkModTime)
-	})
-
-	// Read it again
-	oMetaNew, err := o.Metadata(ctx)
-	require.NoError(t, err)
-
-	// Check that operating on the symlink didn't change the file it was pointing to
-	// See: https://github.com/rclone/rclone/security/advisories/GHSA-hrxh-9w67-g4cv
-	assert.Equal(t, oMeta, oMetaNew, "metadata setting on symlink messed up file")
-
-	// Now run the same tests on the file
-	t.Run("File", func(t *testing.T) {
-		testMetadata(t, r, o, when)
-	})
-}
-
-func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
-	ctx := context.Background()
-	whenRFC := when.Format(time.RFC3339Nano)
-	const dayLength = len("2001-01-01")
-
-	f := r.Flocal.(*Fs)
 	features := f.Features()
 
-	var hasXID, hasAtime, hasBtime, canSetXattrOnLinks bool
+	var hasXID, hasAtime, hasBtime bool
 	switch runtime.GOOS {
 	case "darwin", "freebsd", "netbsd", "linux":
 		hasXID, hasAtime, hasBtime = true, true, true
-		canSetXattrOnLinks = runtime.GOOS != "linux"
 	case "openbsd", "solaris":
 		hasXID, hasAtime = true, true
 	case "windows":
@@ -350,10 +306,6 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 		require.NoError(t, err)
 		assert.Nil(t, m)
 
-		if !canSetXattrOnLinks && o.translatedLink {
-			t.Skip("Skip remainder of test as can't set xattr on symlinks on this OS")
-		}
-
 		inM := fs.Metadata{
 			"potato":  "chips",
 			"cabbage": "soup",
@@ -368,21 +320,18 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 	})
 
 	checkTime := func(m fs.Metadata, key string, when time.Time) {
-		t.Helper()
 		mt, ok := o.parseMetadataTime(m, key)
 		assert.True(t, ok)
 		dt := mt.Sub(when)
 		precision := time.Second
-		assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v want %v got %v", key, dt, precision, mt, when))
+		assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v", key, dt, precision))
 	}
 
 	checkInt := func(m fs.Metadata, key string, base int) int {
-		t.Helper()
 		value, ok := o.parseMetadataInt(m, key, base)
 		assert.True(t, ok)
 		return value
 	}
-
 	t.Run("Read", func(t *testing.T) {
 		m, err := o.Metadata(ctx)
 		require.NoError(t, err)
@@ -392,12 +341,13 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 		checkInt(m, "mode", 8)
 		checkTime(m, "mtime", when)
 
+		assert.Equal(t, len(whenRFC), len(m["mtime"]))
 		assert.Equal(t, whenRFC[:dayLength], m["mtime"][:dayLength])
 
-		if hasAtime && !o.translatedLink { // symlinks generally don't record atime
+		if hasAtime {
 			checkTime(m, "atime", when)
 		}
-		if hasBtime && !o.translatedLink { // symlinks generally don't record btime
+		if hasBtime {
 			checkTime(m, "btime", when)
 		}
 		if hasXID {
@@ -421,10 +371,6 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 			"mode":   "0767",
 			"potato": "wedges",
 		}
-		if !canSetXattrOnLinks && o.translatedLink {
-			// Don't change xattr if not supported on symlinks
-			delete(newM, "potato")
-		}
 		err := o.writeMetadata(newM)
 		require.NoError(t, err)
 
@@ -434,11 +380,7 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 
 		mode := checkInt(m, "mode", 8)
 		if runtime.GOOS != "windows" {
-			expectedMode := 0767
-			if o.translatedLink && runtime.GOOS == "linux" {
-				expectedMode = 0777 // perms of symlinks always read as 0777 on linux
-			}
-			assert.Equal(t, expectedMode, mode&0777, fmt.Sprintf("mode wrong - expecting 0%o got 0%o", expectedMode, mode&0777))
+			assert.Equal(t, 0767, mode&0777, fmt.Sprintf("mode wrong - expecting 0767 got 0%o", mode&0777))
 		}
 
 		checkTime(m, "mtime", newMtime)
@@ -448,7 +390,7 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
 		if haveSetBTime {
 			checkTime(m, "btime", newBtime)
 		}
-		if xattrSupported && (canSetXattrOnLinks || !o.translatedLink) {
+		if xattrSupported {
 			assert.Equal(t, "wedges", m["potato"])
 		}
 	})

backend/local/metadata.go

@@ -105,11 +105,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	}
 	if haveSetBTime {
 		if btimeOK {
-			if o.translatedLink {
-				err = lsetBTime(o.path, btime)
-			} else {
-				err = setBTime(o.path, btime)
-			}
+			err = setBTime(o.path, btime)
 			if err != nil {
 				outErr = fmt.Errorf("failed to set birth (creation) time: %w", err)
 			}
@@ -125,11 +121,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
 			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
 		} else {
-			if o.translatedLink {
-				err = os.Lchown(o.path, uid, gid)
-			} else {
-				err = os.Chown(o.path, uid, gid)
-			}
+			err = os.Chown(o.path, uid, gid)
 			if err != nil {
 				outErr = fmt.Errorf("failed to change ownership: %w", err)
 			}
@@ -140,16 +132,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 		if mode >= 0 {
 			umode := uint(mode)
 			if umode <= math.MaxUint32 {
-				if o.translatedLink {
-					if haveLChmod {
-						err = lChmod(o.path, os.FileMode(umode))
-					} else {
-						fs.Debugf(o, "Unable to set mode %v on a symlink on this OS", os.FileMode(umode))
-						err = nil
-					}
-				} else {
-					err = os.Chmod(o.path, os.FileMode(umode))
-				}
+				err = os.Chmod(o.path, os.FileMode(umode))
 				if err != nil {
 					outErr = fmt.Errorf("failed to change permissions: %w", err)
 				}

backend/local/setbtime.go

@@ -13,9 +13,3 @@ func setBTime(name string, btime time.Time) error {
 	// Does nothing
 	return nil
 }
-
-// lsetBTime changes the birth time of the link passed in
-func lsetBTime(name string, btime time.Time) error {
-	// Does nothing
-	return nil
-}

backend/local/setbtime_windows.go

@@ -9,20 +9,15 @@ import (
 
 const haveSetBTime = true
 
-// setTimes sets any of atime, mtime or btime
-// if link is set it sets a link rather than the target
-func setTimes(name string, atime, mtime, btime time.Time, link bool) (err error) {
+// setBTime sets the birth time of the file passed in
+func setBTime(name string, btime time.Time) (err error) {
 	pathp, err := syscall.UTF16PtrFromString(name)
 	if err != nil {
 		return err
 	}
-	fileFlag := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS)
-	if link {
-		fileFlag |= syscall.FILE_FLAG_OPEN_REPARSE_POINT
-	}
 	h, err := syscall.CreateFile(pathp,
 		syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil,
-		syscall.OPEN_EXISTING, fileFlag, 0)
+		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
 	if err != nil {
 		return err
 	}
@@ -32,28 +27,6 @@ func setTimes(name string, atime, mtime, btime time.Time, link bool) (err error)
 			err = closeErr
 		}
 	}()
-	var patime, pmtime, pbtime *syscall.Filetime
-	if !atime.IsZero() {
-		t := syscall.NsecToFiletime(atime.UnixNano())
-		patime = &t
-	}
-	if !mtime.IsZero() {
-		t := syscall.NsecToFiletime(mtime.UnixNano())
-		pmtime = &t
-	}
-	if !btime.IsZero() {
-		t := syscall.NsecToFiletime(btime.UnixNano())
-		pbtime = &t
-	}
-	return syscall.SetFileTime(h, pbtime, patime, pmtime)
-}
-
-// setBTime sets the birth time of the file passed in
-func setBTime(name string, btime time.Time) (err error) {
-	return setTimes(name, time.Time{}, time.Time{}, btime, false)
-}
-
-// lsetBTime changes the birth time of the link passed in
-func lsetBTime(name string, btime time.Time) error {
-	return setTimes(name, time.Time{}, time.Time{}, btime, true)
+	bFileTime := syscall.NsecToFiletime(btime.UnixNano())
+	return syscall.SetFileTime(h, &bFileTime, nil, nil)
 }

backend/mailru/mailru.go

@@ -68,12 +68,14 @@ var (
 )
 
 // Description of how to authorize
-var oauthConfig = &oauthutil.Config{
+var oauthConfig = &oauth2.Config{
 	ClientID:     api.OAuthClientID,
 	ClientSecret: "",
-	AuthURL:      api.OAuthURL,
-	TokenURL:     api.OAuthURL,
-	AuthStyle:    oauth2.AuthStyleInParams,
+	Endpoint: oauth2.Endpoint{
+		AuthURL:   api.OAuthURL,
+		TokenURL:  api.OAuthURL,
+		AuthStyle: oauth2.AuthStyleInParams,
+	},
 }
 
 // Register with Fs
@@ -436,9 +438,7 @@ func (f *Fs) authorize(ctx context.Context, force bool) (err error) {
 	if err != nil || !tokenIsValid(t) {
 		fs.Infof(f, "Valid token not found, authorizing.")
 		ctx := oauthutil.Context(ctx, f.cli)
-
-		oauth2Conf := oauthConfig.MakeOauth2Config()
-		t, err = oauth2Conf.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password)
+		t, err = oauthConfig.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password)
 	}
 	if err == nil && !tokenIsValid(t) {
 		err = errors.New("invalid token")

backend/onedrive/api/types.go

@@ -202,14 +202,9 @@ type SharingLinkType struct {
 type LinkType string
 
 const (
-	// ViewLinkType (role: read) A view-only sharing link, allowing read-only access.
-	ViewLinkType LinkType = "view"
-	// EditLinkType (role: write) An edit sharing link, allowing read-write access.
-	EditLinkType LinkType = "edit"
-	// EmbedLinkType (role: read) A view-only sharing link that can be used to embed
-	// content into a host webpage. Embed links are not available for OneDrive for
-	// Business or SharePoint.
-	EmbedLinkType LinkType = "embed"
+	ViewLinkType  LinkType = "view"  // ViewLinkType (role: read) A view-only sharing link, allowing read-only access.
+	EditLinkType  LinkType = "edit"  // EditLinkType (role: write) An edit sharing link, allowing read-write access.
+	EmbedLinkType LinkType = "embed" // EmbedLinkType (role: read) A view-only sharing link that can be used to embed content into a host webpage. Embed links are not available for OneDrive for Business or SharePoint.
 )
 
 // LinkScope represents the scope of the link represented by this permission.
@@ -217,12 +212,9 @@ const (
 type LinkScope string
 
 const (
-	// AnonymousScope = Anyone with the link has access, without needing to sign in.
-	// This may include people outside of your organization.
-	AnonymousScope LinkScope = "anonymous"
-	// OrganizationScope = Anyone signed into your organization (tenant) can use the
-	// link to get access. Only available in OneDrive for Business and SharePoint.
-	OrganizationScope LinkScope = "organization"
+	AnonymousScope    LinkScope = "anonymous"    // AnonymousScope = Anyone with the link has access, without needing to sign in. This may include people outside of your organization.
+	OrganizationScope LinkScope = "organization" // OrganizationScope = Anyone signed into your organization (tenant) can use the link to get access. Only available in OneDrive for Business and SharePoint.
+
 )
 
 // PermissionsType provides information about a sharing permission granted for a DriveItem resource.
@@ -244,14 +236,10 @@ type PermissionsType struct {
 type Role string
 
 const (
-	// ReadRole provides the ability to read the metadata and contents of the item.
-	ReadRole Role = "read"
-	// WriteRole provides the ability to read and modify the metadata and contents of the item.
-	WriteRole Role = "write"
-	// OwnerRole represents the owner role for SharePoint and OneDrive for Business.
-	OwnerRole Role = "owner"
-	// MemberRole represents the member role for SharePoint and OneDrive for Business.
-	MemberRole Role = "member"
+	ReadRole   Role = "read"   // ReadRole provides the ability to read the metadata and contents of the item.
+	WriteRole  Role = "write"  // WriteRole provides the ability to read and modify the metadata and contents of the item.
+	OwnerRole  Role = "owner"  // OwnerRole represents the owner role for SharePoint and OneDrive for Business.
+	MemberRole Role = "member" // MemberRole represents the member role for SharePoint and OneDrive for Business.
 )
 
 // PermissionsResponse is the response to the list permissions method

backend/onedrive/onedrive.go

@@ -40,6 +40,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -64,21 +65,14 @@ const (
 
 // Globals
 var (
-
-	// Define the paths used for token operations
-	commonPathPrefix = "/common" // prefix for the paths if tenant isn't known
-	authPath         = "/oauth2/v2.0/authorize"
-	tokenPath        = "/oauth2/v2.0/token"
+	authPath  = "/common/oauth2/v2.0/authorize"
+	tokenPath = "/common/oauth2/v2.0/token"
 
 	scopeAccess             = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "Sites.Read.All", "offline_access"}
 	scopeAccessWithoutSites = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"}
 
-	// When using client credential OAuth flow, scope of .default is required in order
-	// to use the permissions configured for the application within the tenant
-	scopeAccessClientCred = fs.SpaceSepList{".default"}
-
-	// Base config for how to auth
-	oauthConfig = &oauthutil.Config{
+	// Description of how to auth for this app for a business account
+	oauthConfig = &oauth2.Config{
 		Scopes:       scopeAccess,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
@@ -131,7 +125,7 @@ func init() {
 					Help:  "Microsoft Cloud for US Government",
 				}, {
 					Value: regionDE,
-					Help:  "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).",
+					Help:  "Microsoft Cloud Germany",
 				}, {
 					Value: regionCN,
 					Help:  "Azure and Office 365 operated by Vnet Group in China",
@@ -189,14 +183,6 @@ Choose or manually enter a custom space separated list with all scopes, that rcl
 					Help:  "Read and write access to all resources, without the ability to browse SharePoint sites. \nSame as if disable_site_permission was set to true",
 				},
 			},
-		}, {
-			Name: "tenant",
-			Help: `ID of the service principal's tenant. Also called its directory ID.
-
-Set this if using
-- Client Credential flow
-`,
-			Sensitive: true,
 		}, {
 			Name: "disable_site_permission",
 			Help: `Disable the request for Sites.Read.All permission.
@@ -541,54 +527,28 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 	})
 }
 
-// Make the oauth config for the backend
-func makeOauthConfig(ctx context.Context, opt *Options) (*oauthutil.Config, error) {
-	// Copy the default oauthConfig
-	oauthConfig := *oauthConfig
-
-	// Set the scopes
-	oauthConfig.Scopes = opt.AccessScopes
-	if opt.DisableSitePermission {
-		oauthConfig.Scopes = scopeAccessWithoutSites
-	}
-
-	// Construct the auth URLs
-	prefix := commonPathPrefix
-	if opt.Tenant != "" {
-		prefix = "/" + opt.Tenant
-	}
-	oauthConfig.TokenURL = authEndpoint[opt.Region] + prefix + tokenPath
-	oauthConfig.AuthURL = authEndpoint[opt.Region] + prefix + authPath
-
-	// Check to see if we are using client credentials flow
-	if opt.ClientCredentials {
-		// Override scope to .default
-		oauthConfig.Scopes = scopeAccessClientCred
-		if opt.Tenant == "" {
-			return nil, fmt.Errorf("tenant parameter must be set when using %s", config.ConfigClientCredentials)
-		}
-	}
-
-	return &oauthConfig, nil
-}
-
 // Config the backend
-func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.ConfigIn) (*fs.ConfigOut, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-	_, graphURL := getRegionURL(m)
+func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+	region, graphURL := getRegionURL(m)
 
-	// Check to see if this is the start of the state machine execution
-	if conf.State == "" {
-		conf, err := makeOauthConfig(ctx, opt)
+	if config.State == "" {
+		var accessScopes fs.SpaceSepList
+		accessScopesString, _ := m.Get("access_scopes")
+		err := accessScopes.Set(accessScopesString)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to parse access_scopes: %w", err)
+		}
+		oauthConfig.Scopes = []string(accessScopes)
+		disableSitePermission, _ := m.Get("disable_site_permission")
+		if disableSitePermission == "true" {
+			oauthConfig.Scopes = scopeAccessWithoutSites
+		}
+		oauthConfig.Endpoint = oauth2.Endpoint{
+			AuthURL:  authEndpoint[region] + authPath,
+			TokenURL: authEndpoint[region] + tokenPath,
 		}
 		return oauthutil.ConfigOut("choose_type", &oauthutil.Options{
-			OAuth2Config: conf,
+			OAuth2Config: oauthConfig,
 		})
 	}
 
@@ -596,11 +556,9 @@ func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.Config
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure OneDrive: %w", err)
 	}
-
-	// Create a REST client, build on the OAuth client created above
 	srv := rest.NewClient(oAuthClient)
 
-	switch conf.State {
+	switch config.State {
 	case "choose_type":
 		return fs.ConfigChooseExclusiveFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{
 			Value: "onedrive",
@@ -626,7 +584,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.Config
 		}})
 	case "choose_type_done":
 		// Jump to next state according to config chosen
-		return fs.ConfigGoto(conf.Result)
+		return fs.ConfigGoto(config.Result)
 	case "onedrive":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
 			opts: rest.Opts{
@@ -644,22 +602,16 @@ func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.Config
 			},
 		})
 	case "driveid":
-		out, err := fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")
-		if err != nil {
-			return out, err
-		}
-		// Default the drive_id to the previous version in the config
-		out.Option.Default, _ = m.Get("drive_id")
-		return out, nil
+		return fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")
 	case "driveid_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			finalDriveID: conf.Result,
+			finalDriveID: config.Result,
 		})
 	case "siteid":
 		return fs.ConfigInput("siteid_end", "config_siteid", "Site ID")
 	case "siteid_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: conf.Result,
+			siteID: config.Result,
 		})
 	case "url":
 		return fs.ConfigInput("url_end", "config_site_url", `Site URL
@@ -670,7 +622,7 @@ Examples:
 - "https://XXX.sharepoint.com/teams/ID"
 `)
 	case "url_end":
-		siteURL := conf.Result
+		siteURL := config.Result
 		re := regexp.MustCompile(`https://.*\.sharepoint\.com(/.*)`)
 		match := re.FindStringSubmatch(siteURL)
 		if len(match) == 2 {
@@ -685,12 +637,12 @@ Examples:
 		return fs.ConfigInput("path_end", "config_sharepoint_url", `Server-relative URL`)
 	case "path_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			relativePath: conf.Result,
+			relativePath: config.Result,
 		})
 	case "search":
 		return fs.ConfigInput("search_end", "config_search_term", `Search term`)
 	case "search_end":
-		searchTerm := conf.Result
+		searchTerm := config.Result
 		opts := rest.Opts{
 			Method:  "GET",
 			RootURL: graphURL,
@@ -712,10 +664,10 @@ Examples:
 		})
 	case "search_sites":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: conf.Result,
+			siteID: config.Result,
 		})
 	case "driveid_final":
-		finalDriveID := conf.Result
+		finalDriveID := config.Result
 
 		// Test the driveID and get drive type
 		opts := rest.Opts{
@@ -734,12 +686,12 @@ Examples:
 
 		return fs.ConfigConfirm("driveid_final_end", true, "config_drive_ok", fmt.Sprintf("Drive OK?\n\nFound drive %q of type %q\nURL: %s\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL))
 	case "driveid_final_end":
-		if conf.Result == "true" {
+		if config.Result == "true" {
 			return nil, nil
 		}
 		return fs.ConfigGoto("choose_type")
 	}
-	return nil, fmt.Errorf("unknown state %q", conf.State)
+	return nil, fmt.Errorf("unknown state %q", config.State)
 }
 
 // Options defines the configuration for this backend
@@ -750,9 +702,7 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	RootFolderID            string               `config:"root_folder_id"`
 	DisableSitePermission   bool                 `config:"disable_site_permission"`
-	ClientCredentials       bool                 `config:"client_credentials"`
 	AccessScopes            fs.SpaceSepList      `config:"access_scopes"`
-	Tenant                  string               `config:"tenant"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
 	ListChunk               int64                `config:"list_chunk"`
@@ -1040,10 +990,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
-
-	oauthConfig, err := makeOauthConfig(ctx, opt)
-	if err != nil {
-		return nil, err
+	oauthConfig.Scopes = opt.AccessScopes
+	if opt.DisableSitePermission {
+		oauthConfig.Scopes = scopeAccessWithoutSites
+	}
+	oauthConfig.Endpoint = oauth2.Endpoint{
+		AuthURL:  authEndpoint[opt.Region] + authPath,
+		TokenURL: authEndpoint[opt.Region] + tokenPath,
 	}
 
 	client := fshttp.NewClient(ctx)
@@ -1656,7 +1609,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
@@ -1671,18 +1624,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		return nil, fs.ErrorCantCopy
 	}
 
-	err = srcObj.readMetaData(ctx)
+	err := srcObj.readMetaData(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	// Find and remove existing object
-	cleanup, err := operations.RemoveExisting(ctx, f, remote, "server side copy")
-	if err != nil {
-		return nil, err
-	}
-	defer cleanup(&err)
-
 	// Check we aren't overwriting a file on the same remote
 	if srcObj.fs == f {
 		srcPath := srcObj.rootPath()
@@ -2610,11 +2556,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.New("can't upload content to a OneNote file")
 	}
 
-	// Only start the renewer if we have a valid one
-	if o.fs.tokenRenewer != nil {
-		o.fs.tokenRenewer.Start()
-		defer o.fs.tokenRenewer.Stop()
-	}
+	o.fs.tokenRenewer.Start()
+	defer o.fs.tokenRenewer.Stop()
 
 	size := src.Size()
 

backend/onedrive/onedrive_internal_test.go

@@ -215,11 +215,11 @@ func (f *Fs) TestDirectoryMetadata(t *testing.T, r *fstest.Run) {
 	compareDirMeta(expectedMeta, actualMeta, false)
 
 	// modtime
-	fstest.AssertTimeEqualWithPrecision(t, newDst.Remote(), t1, newDst.ModTime(ctx), f.Precision())
+	assert.Equal(t, t1.Truncate(f.Precision()), newDst.ModTime(ctx))
 	// try changing it and re-check it
 	newDst, err = operations.SetDirModTime(ctx, f, newDst, "", t2)
 	assert.NoError(t, err)
-	fstest.AssertTimeEqualWithPrecision(t, newDst.Remote(), t2, newDst.ModTime(ctx), f.Precision())
+	assert.Equal(t, t2.Truncate(f.Precision()), newDst.ModTime(ctx))
 	// ensure that f.DirSetModTime also works
 	err = f.DirSetModTime(ctx, "subdir", t3)
 	assert.NoError(t, err)
@@ -227,7 +227,7 @@ func (f *Fs) TestDirectoryMetadata(t *testing.T, r *fstest.Run) {
 	assert.NoError(t, err)
 	entries.ForDir(func(dir fs.Directory) {
 		if dir.Remote() == "subdir" {
-			fstest.AssertTimeEqualWithPrecision(t, dir.Remote(), t3, dir.ModTime(ctx), f.Precision())
+			assert.True(t, t3.Truncate(f.Precision()).Equal(dir.ModTime(ctx)), fmt.Sprintf("got %v", dir.ModTime(ctx)))
 		}
 	})
 

backend/oracleobjectstorage/multipart.go

@@ -22,7 +22,6 @@ import (
 	"github.com/oracle/oci-go-sdk/v65/objectstorage"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
-	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 )
 
@@ -184,9 +183,6 @@ func (w *objectChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, rea
 			if ossPartNumber <= 8 {
 				return shouldRetry(ctx, resp.HTTPResponse(), err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/oracleobjectstorage/options.go

@@ -106,9 +106,9 @@ func newOptions() []fs.Option {
 		Sensitive: true,
 	}, {
 		Name:      "compartment",
-		Help:      "Specify compartment OCID, if you need to list buckets.\n\nList objects works without compartment OCID.",
+		Help:      "Object storage compartment OCID",
 		Provider:  "!no_auth",
-		Required:  false,
+		Required:  true,
 		Sensitive: true,
 	}, {
 		Name:     "region",

backend/pcloud/pcloud.go

@@ -48,10 +48,12 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		Scopes:  nil,
-		AuthURL: "https://my.pcloud.com/oauth2/authorize",
-		// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
+	oauthConfig = &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL: "https://my.pcloud.com/oauth2/authorize",
+			// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -59,8 +61,8 @@ var (
 )
 
 // Update the TokenURL with the actual hostname
-func updateTokenURL(oauthConfig *oauthutil.Config, hostname string) {
-	oauthConfig.TokenURL = "https://" + hostname + "/oauth2_token"
+func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
+	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
 }
 
 // Register with Fs
@@ -77,7 +79,7 @@ func init() {
 				fs.Errorf(nil, "Failed to read config: %v", err)
 			}
 			updateTokenURL(oauthConfig, optc.Hostname)
-			checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error {
+			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
 				if auth == nil || auth.Form == nil {
 					return errors.New("form not found in response")
 				}
@@ -397,15 +399,14 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 	if err != nil {
 		return nil, fmt.Errorf("open file: %w", err)
 	}
-	if _, err := fileClose(ctx, client, f.pacer, openResult.FileDescriptor); err != nil {
-		return nil, fmt.Errorf("close file: %w", err)
-	}
 
 	writer := &writerAt{
 		ctx:    ctx,
+		client: client,
 		fs:     f,
 		size:   size,
 		remote: remote,
+		fd:     openResult.FileDescriptor,
 		fileID: openResult.Fileid,
 	}
 
@@ -424,7 +425,7 @@ func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) {
 	})
 	// Set our own http client in the context
 	ctx = oauthutil.Context(ctx, baseClient)
-	// create a new oauth client, reuse the token source
+	// create a new oauth client, re-use the token source
 	oAuthClient := oauth2.NewClient(ctx, f.ts)
 	return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil
 }

backend/pcloud/writer_at.go

@@ -18,14 +18,21 @@ import (
 // writerAt implements fs.WriterAtCloser, adding the OpenWrtierAt feature to pcloud.
 type writerAt struct {
 	ctx    context.Context
+	client *rest.Client
 	fs     *Fs
 	size   int64
 	remote string
+	fd     int64
 	fileID int64
 }
 
 // Close implements WriterAt.Close.
 func (c *writerAt) Close() error {
+	// close fd
+	if _, err := c.fileClose(c.ctx); err != nil {
+		return fmt.Errorf("close fd: %w", err)
+	}
+
 	// Avoiding race conditions: Depending on the tcp connection, there might be
 	// caching issues when checking the size immediately after write.
 	// Hence we try avoiding them by checking the resulting size on a different connection.
@@ -65,18 +72,8 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	inSHA1Bytes := sha1.Sum(buffer)
 	inSHA1 := hex.EncodeToString(inSHA1Bytes[:])
 
-	client, err := c.fs.newSingleConnClient(c.ctx)
-	if err != nil {
-		return 0, fmt.Errorf("create client: %w", err)
-	}
-
-	openResult, err := fileOpen(c.ctx, client, c.fs, c.fileID)
-	if err != nil {
-		return 0, fmt.Errorf("open file: %w", err)
-	}
-
 	// get target hash
-	outChecksum, err := fileChecksum(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, int64(contentLength))
+	outChecksum, err := c.fileChecksum(c.ctx, offset, int64(contentLength))
 	if err != nil {
 		return 0, err
 	}
@@ -92,15 +89,10 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	}
 
 	// upload buffer with offset if necessary
-	if _, err := filePWrite(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, buffer); err != nil {
+	if _, err := c.filePWrite(c.ctx, offset, buffer); err != nil {
 		return 0, err
 	}
 
-	// close fd
-	if _, err := fileClose(c.ctx, client, c.fs.pacer, openResult.FileDescriptor); err != nil {
-		return contentLength, fmt.Errorf("close fd: %w", err)
-	}
-
 	return contentLength, nil
 }
 
@@ -133,40 +125,11 @@ func fileOpenNew(ctx context.Context, c *rest.Client, srcFs *Fs, directoryID, fi
 	return result, nil
 }
 
-// Call pcloud file_open using fileid with O_WRITE flags, see [API Doc.]
-// [API Doc]: https://docs.pcloud.com/methods/fileops/file_open.html
-func fileOpen(ctx context.Context, c *rest.Client, srcFs *Fs, fileID int64) (*api.FileOpenResponse, error) {
-	opts := rest.Opts{
-		Method:           "PUT",
-		Path:             "/file_open",
-		Parameters:       url.Values{},
-		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
-		ExtraHeaders: map[string]string{
-			"Connection": "keep-alive",
-		},
-	}
-	opts.Parameters.Set("fileid", strconv.FormatInt(fileID, 10))
-	opts.Parameters.Set("flags", "0x0002") // O_WRITE
-
-	result := &api.FileOpenResponse{}
-	err := srcFs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.CallJSON(ctx, &opts, nil, result)
-		err = result.Error.Update(err)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("open new file descriptor: %w", err)
-	}
-	return result, nil
-}
-
 // Call pcloud file_checksum, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_checksum.html
-func fileChecksum(
+func (c *writerAt) fileChecksum(
 	ctx context.Context,
-	client *rest.Client,
-	pacer *fs.Pacer,
-	fd, offset, count int64,
+	offset, count int64,
 ) (*api.FileChecksumResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
@@ -177,29 +140,26 @@ func fileChecksum(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 	opts.Parameters.Set("count", strconv.FormatInt(count, 10))
 
 	result := &api.FileChecksumResponse{}
-	err := pacer.CallNoRetry(func() (bool, error) {
-		resp, err := client.CallJSON(ctx, &opts, nil, result)
+	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", fd, offset, count, err)
+		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", c.fd, offset, count, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_pwrite, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_pwrite.html
-func filePWrite(
+func (c *writerAt) filePWrite(
 	ctx context.Context,
-	client *rest.Client,
-	pacer *fs.Pacer,
-	fd int64,
 	offset int64,
 	buf []byte,
 ) (*api.FilePWriteResponse, error) {
@@ -216,29 +176,24 @@ func filePWrite(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 
 	result := &api.FilePWriteResponse{}
-	err := pacer.CallNoRetry(func() (bool, error) {
-		resp, err := client.CallJSON(ctx, &opts, nil, result)
+	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, fd, offset, err)
+		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, c.fd, offset, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_close, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_close.html
-func fileClose(
-	ctx context.Context,
-	client *rest.Client,
-	pacer *fs.Pacer,
-	fd int64,
-) (*api.FileCloseResponse, error) {
+func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
 		Path:             "/file_close",
@@ -246,11 +201,11 @@ func fileClose(
 		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
 		Close:            true,
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
 
 	result := &api.FileCloseResponse{}
-	err := pacer.CallNoRetry(func() (bool, error) {
-		resp, err := client.CallJSON(ctx, &opts, nil, result)
+	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})

backend/pikpak/pikpak.go

@@ -82,11 +82,13 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		Scopes:      nil,
-		AuthURL:     "https://user.mypikpak.com/v1/auth/signin",
-		TokenURL:    "https://user.mypikpak.com/v1/auth/token",
-		AuthStyle:   oauth2.AuthStyleInParams,
+	oauthConfig = &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:   "https://user.mypikpak.com/v1/auth/signin",
+			TokenURL:  "https://user.mypikpak.com/v1/auth/token",
+			AuthStyle: oauth2.AuthStyleInParams,
+		},
 		ClientID:    clientID,
 		RedirectURL: oauthutil.RedirectURL,
 	}
@@ -213,11 +215,6 @@ Fill in for rclone to use a non root folder as its starting point.
 			Default:  false,
 			Help:     "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
 			Advanced: true,
-		}, {
-			Name:     "no_media_link",
-			Default:  false,
-			Help:     "Use original file links instead of media links.\n\nThis avoids issues caused by invalid media links, but may reduce download speeds.",
-			Advanced: true,
 		}, {
 			Name:     "hash_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate hash if required.",
@@ -291,7 +288,6 @@ type Options struct {
 	RootFolderID        string               `config:"root_folder_id"`
 	UseTrash            bool                 `config:"use_trash"`
 	TrashedOnly         bool                 `config:"trashed_only"`
-	NoMediaLink         bool                 `config:"no_media_link"`
 	HashMemoryThreshold fs.SizeSuffix        `config:"hash_memory_limit"`
 	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
 	UploadConcurrency   int                  `config:"upload_concurrency"`
@@ -565,7 +561,6 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		if strings.Contains(err.Error(), "invalid_grant") {
 			return f, f.reAuthorize(ctx)
 		}
-		return nil, err
 	}
 
 	return f, nil
@@ -1581,14 +1576,15 @@ func (o *Object) setMetaData(info *api.File) (err error) {
 	o.md5sum = info.Md5Checksum
 	if info.Links.ApplicationOctetStream != nil {
 		o.link = info.Links.ApplicationOctetStream
-		if !o.fs.opt.NoMediaLink {
-			if fid := parseFileID(o.link.URL); fid != "" {
-				for _, media := range info.Medias {
-					if media.Link != nil && parseFileID(media.Link.URL) == fid {
-						fs.Debugf(o, "Using a media link")
-						o.link = media.Link
-						break
-					}
+		if fid := parseFileID(o.link.URL); fid != "" {
+			for mid, media := range info.Medias {
+				if media.Link == nil {
+					continue
+				}
+				if mfid := parseFileID(media.Link.URL); fid == mfid {
+					fs.Debugf(o, "Using a media link from Medias[%d]", mid)
+					o.link = media.Link
+					break
 				}
 			}
 		}

backend/premiumizeme/premiumizeme.go

@@ -43,6 +43,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -58,10 +59,12 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		Scopes:       nil,
-		AuthURL:      "https://www.premiumize.me/authorize",
-		TokenURL:     "https://www.premiumize.me/token",
+	oauthConfig = &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://www.premiumize.me/authorize",
+			TokenURL: "https://www.premiumize.me/token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/putio/fs.go

@@ -572,17 +572,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 	if err != nil {
 		return nil, err
 	}
-
-	// We have successfully copied the file to random name
-	// Check to see if file already exists first and delete it if so
-	existingObj, err := f.NewObject(ctx, remote)
-	if err == nil {
-		err = existingObj.Remove(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("server side copy: failed to remove existing file: %w", err)
-		}
-	}
-
 	err = f.pacer.Call(func() (bool, error) {
 		params := url.Values{}
 		params.Set("file_id", strconv.FormatInt(resp.File.ID, 10))

backend/putio/putio.go

@@ -13,6 +13,7 @@ import (
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
+	"golang.org/x/oauth2"
 )
 
 /*
@@ -40,10 +41,12 @@ const (
 
 var (
 	// Description of how to auth for this app
-	putioConfig = &oauthutil.Config{
-		Scopes:       []string{},
-		AuthURL:      "https://api.put.io/v2/oauth2/authenticate",
-		TokenURL:     "https://api.put.io/v2/oauth2/access_token",
+	putioConfig = &oauth2.Config{
+		Scopes: []string{},
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://api.put.io/v2/oauth2/authenticate",
+			TokenURL: "https://api.put.io/v2/oauth2/access_token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneObscuredClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,

backend/s3/s3.go

@@ -934,67 +934,34 @@ func init() {
 				Help:  "The default endpoint\nIran",
 			}},
 		}, {
-			// Linode endpoints: https://techdocs.akamai.com/cloud-computing/docs/object-storage-product-limits#supported-endpoint-types-by-region
+			// Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint
 			Name:     "endpoint",
 			Help:     "Endpoint for Linode Object Storage API.",
 			Provider: "Linode",
 			Examples: []fs.OptionExample{{
-				Value: "nl-ams-1.linodeobjects.com",
-				Help:  "Amsterdam (Netherlands), nl-ams-1",
-			}, {
 				Value: "us-southeast-1.linodeobjects.com",
 				Help:  "Atlanta, GA (USA), us-southeast-1",
-			}, {
-				Value: "in-maa-1.linodeobjects.com",
-				Help:  "Chennai (India), in-maa-1",
 			}, {
 				Value: "us-ord-1.linodeobjects.com",
 				Help:  "Chicago, IL (USA), us-ord-1",
 			}, {
 				Value: "eu-central-1.linodeobjects.com",
 				Help:  "Frankfurt (Germany), eu-central-1",
-			}, {
-				Value: "id-cgk-1.linodeobjects.com",
-				Help:  "Jakarta (Indonesia), id-cgk-1",
-			}, {
-				Value: "gb-lon-1.linodeobjects.com",
-				Help:  "London 2 (Great Britain), gb-lon-1",
-			}, {
-				Value: "us-lax-1.linodeobjects.com",
-				Help:  "Los Angeles, CA (USA), us-lax-1",
-			}, {
-				Value: "es-mad-1.linodeobjects.com",
-				Help:  "Madrid (Spain), es-mad-1",
-			}, {
-				Value: "au-mel-1.linodeobjects.com",
-				Help:  "Melbourne (Australia), au-mel-1",
-			}, {
-				Value: "us-mia-1.linodeobjects.com",
-				Help:  "Miami, FL (USA), us-mia-1",
 			}, {
 				Value: "it-mil-1.linodeobjects.com",
 				Help:  "Milan (Italy), it-mil-1",
 			}, {
 				Value: "us-east-1.linodeobjects.com",
 				Help:  "Newark, NJ (USA), us-east-1",
-			}, {
-				Value: "jp-osa-1.linodeobjects.com",
-				Help:  "Osaka (Japan), jp-osa-1",
 			}, {
 				Value: "fr-par-1.linodeobjects.com",
 				Help:  "Paris (France), fr-par-1",
-			}, {
-				Value: "br-gru-1.linodeobjects.com",
-				Help:  "São Paulo (Brazil), br-gru-1",
 			}, {
 				Value: "us-sea-1.linodeobjects.com",
 				Help:  "Seattle, WA (USA), us-sea-1",
 			}, {
 				Value: "ap-south-1.linodeobjects.com",
-				Help:  "Singapore, ap-south-1",
-			}, {
-				Value: "sg-sin-1.linodeobjects.com",
-				Help:  "Singapore 2, sg-sin-1",
+				Help:  "Singapore ap-south-1",
 			}, {
 				Value: "se-sto-1.linodeobjects.com",
 				Help:  "Stockholm (Sweden), se-sto-1",
@@ -1376,7 +1343,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1389,10 +1356,6 @@ func init() {
 				Value:    "sfo3.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces San Francisco 3",
 				Provider: "DigitalOcean",
-			}, {
-				Value:    "sfo2.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces San Francisco 2",
-				Provider: "DigitalOcean",
 			}, {
 				Value:    "fra1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Frankfurt 1",
@@ -1409,18 +1372,6 @@ func init() {
 				Value:    "sgp1.digitaloceanspaces.com",
 				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
-			}, {
-				Value:    "lon1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces London 1",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "tor1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Toronto 1",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "blr1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Bangalore 1",
-				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
@@ -1525,6 +1476,14 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
+			}, {
+				Value:    "br-se1.magaluobjects.com",
+				Help:     "Magalu BR Southeast 1 endpoint",
+				Provider: "Magalu",
+			}, {
+				Value:    "br-ne1.magaluobjects.com",
+				Help:     "Magalu BR Northeast 1 endpoint",
+				Provider: "Magalu",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -2097,7 +2056,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "One Zone Infrequent Access storage class",
 			}, {
 				Value: "GLACIER",
-				Help:  "Glacier Flexible Retrieval storage class",
+				Help:  "Glacier storage class",
 			}, {
 				Value: "DEEP_ARCHIVE",
 				Help:  "Glacier Deep Archive storage class",
@@ -2163,16 +2122,13 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Help:  "Standard storage class",
 			}},
 		}, {
-			// Mapping from here: https://docs.magalu.cloud/docs/storage/object-storage/Classes-de-Armazenamento/standard
+			// Mapping from here: #todo
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in Magalu.",
 			Provider: "Magalu",
 			Examples: []fs.OptionExample{{
 				Value: "STANDARD",
 				Help:  "Standard storage class",
-			}, {
-				Value: "GLACIER_IR",
-				Help:  "Glacier Instant Retrieval storage class",
 			}},
 		}, {
 			// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
@@ -3388,7 +3344,7 @@ func setQuirks(opt *Options) {
 		listObjectsV2         = true // Always use ListObjectsV2 instead of ListObjects
 		virtualHostStyle      = true // Use bucket.provider.com instead of putting the bucket in the URL
 		urlEncodeListings     = true // URL encode the listings to help with control characters
-		useMultipartEtag      = true // Set if Etags for multipart uploads are compatible with AWS
+		useMultipartEtag      = true // Set if Etags for multpart uploads are compatible with AWS
 		useAcceptEncodingGzip = true // Set Accept-Encoding: gzip
 		mightGzip             = true // assume all providers might use content encoding gzip until proven otherwise
 		useAlreadyExists      = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
@@ -5910,25 +5866,6 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 	return resp.Body, err
 }
 
-// middleware to stop the SDK adding `Accept-Encoding: identity`
-func removeDisableGzip() func(*middleware.Stack) error {
-	return func(stack *middleware.Stack) error {
-		_, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")
-		return err
-	}
-}
-
-// middleware to set Accept-Encoding to how we want it
-//
-// This make sure we download compressed files as-is from all platforms
-func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {
-	APIOptions = append(APIOptions, removeDisableGzip())
-	if f.opt.UseAcceptEncodingGzip.Value {
-		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
-	}
-	return APIOptions
-}
-
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	bucket, bucketPath := o.split()
@@ -5962,8 +5899,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	var APIOptions []func(*middleware.Stack) error
 
-	// Set the SDK to always download compressed files as-is
-	APIOptions = append(APIOptions, o.fs.acceptEncoding()...)
+	// Override the automatic decompression in the transport to
+	// download compressed files as-is
+	if o.fs.opt.UseAcceptEncodingGzip.Value {
+		APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
+	}
 
 	for _, option := range options {
 		switch option.(type) {
@@ -6101,7 +6041,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 			if mOut == nil {
 				err = fserrors.RetryErrorf("internal error: no info from multipart upload")
 			} else if mOut.UploadId == nil {
-				err = fserrors.RetryErrorf("internal error: no UploadId in multipart upload: %#v", *mOut)
+				err = fserrors.RetryErrorf("internal error: no UploadId in multpart upload: %#v", *mOut)
 			}
 		}
 		return f.shouldRetry(ctx, err)
@@ -6114,8 +6054,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 		chunkSize:            int64(chunkSize),
 		size:                 size,
 		f:                    f,
-		bucket:               ui.req.Bucket,
-		key:                  ui.req.Key,
+		bucket:               mOut.Bucket,
+		key:                  mOut.Key,
 		uploadID:             mOut.UploadId,
 		multiPartUploadInput: &mReq,
 		completedParts:       make([]types.CompletedPart, 0),
@@ -6219,9 +6159,6 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}

backend/s3/s3_test.go

@@ -23,20 +23,14 @@ func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) {
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
-	opt := &fstests.Opt{
+	fstests.Run(t, &fstests.Opt{
 		RemoteName:  "TestS3:",
 		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"STANDARD"},
+		TiersToTest: []string{"STANDARD", "STANDARD_IA"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: minChunkSize,
 		},
-	}
-	// Test wider range of tiers on AWS
-	if *fstest.RemoteName == "" || *fstest.RemoteName == "TestS3:" {
-		opt.TiersToTest = []string{"STANDARD", "STANDARD_IA"}
-	}
-	fstests.Run(t, opt)
-
+	})
 }
 
 func TestIntegration2(t *testing.T) {

backend/sftp/sftp.go

@@ -99,11 +99,6 @@ Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
 			Sensitive:  true,
-		}, {
-			Name: "pubkey",
-			Help: `SSH public certificate for public certificate based authentication.
-Set this if you have a signed certificate you want to use for authentication.
-If specified will override pubkey_file.`,
 		}, {
 			Name: "pubkey_file",
 			Help: `Optional path to public key file.
@@ -516,7 +511,6 @@ type Options struct {
 	KeyPem                  string          `config:"key_pem"`
 	KeyFile                 string          `config:"key_file"`
 	KeyFilePass             string          `config:"key_file_pass"`
-	PubKey                  string          `config:"pubkey"`
 	PubKeyFile              string          `config:"pubkey_file"`
 	KnownHostsFile          string          `config:"known_hosts_file"`
 	KeyUseAgent             bool            `config:"key_use_agent"`
@@ -1003,21 +997,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 
 		// If a public key has been specified then use that
-		if pubkeyFile != "" || opt.PubKey != "" {
-			pubKeyRaw := []byte(opt.PubKey)
-			// Use this error if public key is provided inline and is not a certificate
-			// if public key file is provided instead, use the err in the if block
-			notACertError := errors.New("public key provided is not a certificate: " + opt.PubKey)
-			if opt.PubKey == "" {
-				notACertError = errors.New("public key file is not a certificate file: " + pubkeyFile)
-				err := error(nil)
-				pubKeyRaw, err = os.ReadFile(pubkeyFile)
-				if err != nil {
-					return nil, fmt.Errorf("unable to read cert file: %w", err)
-				}
+		if pubkeyFile != "" {
+			certfile, err := os.ReadFile(pubkeyFile)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read cert file: %w", err)
 			}
 
-			pk, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyRaw)
+			pk, _, _, _, err := ssh.ParseAuthorizedKey(certfile)
 			if err != nil {
 				return nil, fmt.Errorf("unable to parse cert file: %w", err)
 			}
@@ -1031,7 +1017,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// knows everything it needs.
 			cert, ok := pk.(*ssh.Certificate)
 			if !ok {
-				return nil, notACertError
+				return nil, errors.New("public key file is not a certificate file: " + pubkeyFile)
 			}
 			pubsigner, err := ssh.NewCertSigner(cert, signer)
 			if err != nil {
@@ -2101,10 +2087,10 @@ func (file *objectReader) Read(p []byte) (n int, err error) {
 
 // Close a reader of a remote sftp file
 func (file *objectReader) Close() (err error) {
-	// Close the pipeReader so writes to the pipeWriter fail
-	_ = file.pipeReader.Close()
 	// Close the sftpFile - this will likely cause the WriteTo to error
 	err = file.sftpFile.Close()
+	// Close the pipeReader so writes to the pipeWriter fail
+	_ = file.pipeReader.Close()
 	// Wait for the background process to finish
 	<-file.done
 	// Show connection no longer in use

backend/sharefile/sharefile.go

@@ -97,6 +97,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -114,11 +115,13 @@ const (
 )
 
 // Generate a new oauth2 config which we will update when we know the TokenURL
-func newOauthConfig(tokenURL string) *oauthutil.Config {
-	return &oauthutil.Config{
-		Scopes:       nil,
-		AuthURL:      "https://secure.sharefile.com/oauth/authorize",
-		TokenURL:     tokenURL,
+func newOauthConfig(tokenURL string) *oauth2.Config {
+	return &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://secure.sharefile.com/oauth/authorize",
+			TokenURL: tokenURL,
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectPublicSecureURL,
@@ -133,7 +136,7 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
 			oauthConfig := newOauthConfig("")
-			checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error {
+			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
 				if auth == nil || auth.Form == nil {
 					return errors.New("endpoint not found in response")
 				}
@@ -144,7 +147,7 @@ func init() {
 				}
 				endpoint := "https://" + subdomain + "." + apicp
 				m.Set("endpoint", endpoint)
-				oauthConfig.TokenURL = endpoint + tokenPath
+				oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
 				return nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{

backend/smb/smb.go

@@ -601,10 +601,9 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	}
 
 	fi, err := cn.smbShare.Stat(reqDir)
-	if err != nil {
-		return fmt.Errorf("SetModTime: stat: %w", err)
+	if err == nil {
+		o.statResult = fi
 	}
-	o.statResult = fi
 	return err
 }
 
@@ -686,6 +685,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 	defer func() {
+		o.statResult, _ = cn.smbShare.Stat(filename)
 		o.fs.putConnection(&cn)
 	}()
 
@@ -723,7 +723,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update Close failed: %w", err)
 	}
 
-	// Set the modified time and also o.statResult
+	// Set the modified time
 	err = o.SetModTime(ctx, src.ModTime(ctx))
 	if err != nil {
 		return fmt.Errorf("Update SetModTime failed: %w", err)

backend/sugarsync/sugarsync.go

@@ -35,7 +35,6 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
@@ -120,7 +119,7 @@ func init() {
 				srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) //  FIXME
 
 				// FIXME
-				// err = f.pacer.Call(func() (bool, error) {
+				//err = f.pacer.Call(func() (bool, error) {
 				resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
 				//	return shouldRetry(ctx, resp, err)
 				//})
@@ -327,7 +326,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
 func (f *Fs) getAuthToken(ctx context.Context) error {
 	fs.Debugf(f, "Renewing token")
 
-	authRequest := api.TokenAuthRequest{
+	var authRequest = api.TokenAuthRequest{
 		AccessKeyID:      withDefault(f.opt.AccessKeyID, accessKeyID),
 		PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)),
 		RefreshToken:     f.opt.RefreshToken,
@@ -509,7 +508,7 @@ func errorHandler(resp *http.Response) (err error) {
 		return fmt.Errorf("error reading error out of body: %w", err)
 	}
 	match := findError.FindSubmatch(body)
-	if len(match) < 2 || len(match[1]) == 0 {
+	if match == nil || len(match) < 2 || len(match[1]) == 0 {
 		return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
 	}
 	return fmt.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1])
@@ -552,7 +551,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
-	// fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
+	//fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
 	// Find the leaf in pathID
 	found, err = f.listAll(ctx, pathID, nil, func(item *api.Collection) bool {
 		if strings.EqualFold(item.Name, leaf) {
@@ -868,13 +867,13 @@ func (f *Fs) Precision() time.Duration {
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	err = srcObj.readMetaData(ctx)
+	err := srcObj.readMetaData(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -891,13 +890,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		return nil, err
 	}
 
-	// Find and remove existing object
-	cleanup, err := operations.RemoveExisting(ctx, f, remote, "server side copy")
-	if err != nil {
-		return nil, err
-	}
-	defer cleanup(&err)
-
 	// Copy the object
 	opts := rest.Opts{
 		Method:     "POST",

backend/webdav/webdav.go

@@ -161,24 +161,7 @@ Set to 0 to disable chunked uploading.
 			Default:  false,
 		},
 			fshttp.UnixSocketConfig,
-			{
-				Name: "auth_redirect",
-				Help: `Preserve authentication on redirect.
-
-If the server redirects rclone to a new domain when it is trying to
-read a file then normally rclone will drop the Authorization: header
-from the request.
-
-This is standard security practice to avoid sending your credentials
-to an unknown webserver.
-
-However this is desirable in some circumstances. If you are getting
-an error like "401 Unauthorized" when rclone is attempting to read
-files from the webdav server then you can try this option.
-`,
-				Advanced: true,
-				Default:  false,
-			}},
+		},
 	})
 }
 
@@ -197,7 +180,6 @@ type Options struct {
 	ExcludeShares      bool                 `config:"owncloud_exclude_shares"`
 	ExcludeMounts      bool                 `config:"owncloud_exclude_mounts"`
 	UnixSocket         string               `config:"unix_socket"`
-	AuthRedirect       bool                 `config:"auth_redirect"`
 }
 
 // Fs represents a remote webdav
@@ -1474,7 +1456,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		ExtraHeaders: map[string]string{
 			"Depth": "0",
 		},
-		AuthRedirect: o.fs.opt.AuthRedirect, // allow redirects to preserve Auth
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)

backend/yandex/yandex.go

@@ -22,13 +22,13 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 // oAuth
@@ -46,9 +46,11 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		AuthURL:      "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
-		TokenURL:     "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
+			TokenURL: "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -711,7 +713,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
@@ -719,21 +721,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	dstPath := f.filePath(remote)
-	err = f.mkParentDirs(ctx, dstPath)
+	err := f.mkParentDirs(ctx, dstPath)
 	if err != nil {
 		return nil, err
 	}
-
-	// Find and remove existing object
-	//
-	// Note that the overwrite flag doesn't seem to work for server side copy
-	cleanup, err := operations.RemoveExisting(ctx, f, remote, "server side copy")
-	if err != nil {
-		return nil, err
-	}
-	defer cleanup(&err)
-
 	err = f.copyOrMove(ctx, "copy", srcObj.filePath(), dstPath, false)
+
 	if err != nil {
 		return nil, fmt.Errorf("couldn't copy file: %w", err)
 	}

backend/zoho/zoho.go

@@ -47,7 +47,7 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
+	oauthConfig = &oauth2.Config{
 		Scopes: []string{
 			"aaaserver.profile.read",
 			"WorkDrive.team.READ",
@@ -55,10 +55,11 @@ var (
 			"WorkDrive.files.ALL",
 			"ZohoFiles.files.ALL",
 		},
-
-		AuthURL:      "https://accounts.zoho.eu/oauth/v2/auth",
-		TokenURL:     "https://accounts.zoho.eu/oauth/v2/token",
-		AuthStyle:    oauth2.AuthStyleInParams,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:   "https://accounts.zoho.eu/oauth/v2/auth",
+			TokenURL:  "https://accounts.zoho.eu/oauth/v2/token",
+			AuthStyle: oauth2.AuthStyleInParams,
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -275,8 +276,8 @@ func setupRegion(m configmap.Mapper) error {
 	downloadURL = fmt.Sprintf("https://download.zoho.%s/v1/workdrive", region)
 	uploadURL = fmt.Sprintf("https://upload.zoho.%s/workdrive-api/v1", region)
 	accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
-	oauthConfig.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
-	oauthConfig.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
+	oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
+	oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
 	return nil
 }
 

bin/backend-versions.sh

@@ -7,11 +7,11 @@ for backend in $( find backend -maxdepth 1 -type d ); do
         continue
     fi
     
-    commit=$(git log --oneline -- $backend | tail -n 1 | cut -d' ' -f1)
+    commit=$(git log --oneline -- $backend | tail -1 | cut -d' ' -f1)
     if [ "$commit" == "" ]; then
-        commit=$(git log --oneline -- backend/$backend | tail -n 1 | cut -d' ' -f1)
+        commit=$(git log --oneline -- backend/$backend | tail -1 | cut -d' ' -f1)
     fi
-    version=$(git tag --contains $commit | grep ^v | sort -n | head -n 1)
+    version=$(git tag --contains $commit | grep ^v | sort -n | head -1)
     echo $backend $version
     sed -i~ "4i versionIntroduced: \"$version\"" docs/content/${backend}.md
 done

bin/make_manual.py

@@ -7,7 +7,6 @@ conversion into man pages etc.
 import os
 import re
 import time
-import subprocess
 from datetime import datetime
 
 docpath = "docs/content"
@@ -36,7 +35,6 @@ docs = [
     "box.md",
     "cache.md",
     "chunker.md",
-    "cloudinary.md",
     "sharefile.md",
     "crypt.md",
     "compress.md",
@@ -193,23 +191,13 @@ def main():
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
     build_date = datetime.utcfromtimestamp(
             int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
-    help_output = subprocess.check_output(["rclone", "help"]).decode("utf-8")
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-# NAME
-
-rclone - manage files on cloud storage
-
-# SYNOPSIS
-
-```
-%s
-```
-""" % (build_date.strftime("%b %d, %Y"), help_output))
+""" % build_date.strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

bin/not-in-stable.go

@@ -29,7 +29,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 	cmd := exec.Command("git", "log", "--oneline", from+".."+to)
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	logMap = map[string]string{}
 	logs = []string{}
@@ -39,7 +39,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 		}
 		match := logRe.FindSubmatch(line)
 		if match == nil {
-			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+			log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 		}
 		var hash, logMessage = string(match[1]), string(match[2])
 		logMap[logMessage] = hash
@@ -52,12 +52,12 @@ func main() {
 	flag.Parse()
 	args := flag.Args()
 	if len(args) != 0 {
-		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	// v1.54.0
 	versionBytes, err := os.ReadFile("VERSION")
 	if err != nil {
-		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+		log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	}
 	if versionBytes[0] == 'v' {
 		versionBytes = versionBytes[1:]
@@ -65,7 +65,7 @@ func main() {
 	versionBytes = bytes.TrimSpace(versionBytes)
 	semver := semver.New(string(versionBytes))
 	stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1)
-	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log.
+	log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log.
 	masterMap, masterLogs := readCommits(stable+".0", "master")
 	stableMap, _ := readCommits(stable+".0", stable+"-stable")
 	for _, logMessage := range masterLogs {

bin/test_backend_sizes.py

@@ -7,18 +7,15 @@ Run with no arguments to test all backends or a supply a list of
 backends to test.
 """
 
-import os
-import re
-import sys
-import subprocess
-
 all_backends = "backend/all/all.go"
 
 # compile command which is more or less like the production builds
 compile_command = ["go", "build", "--ldflags", "-s", "-trimpath"]
 
-# disable CGO as that makes a lot of difference to binary size
-os.environ["CGO_ENABLED"]="0"
+import os
+import re
+import sys
+import subprocess
 
 match_backend = re.compile(r'"github.com/rclone/rclone/backend/(.*?)"')
 
@@ -46,9 +43,6 @@ def write_all(orig_all, backend):
             # Comment out line matching backend
             if match and match.group(1) == backend:
                 line = "// " + line
-            # s3 and pikpak depend on each other
-            if backend == "s3" and "pikpak" in line:
-                line = "// " + line
             fd.write(line+"\n")
 
 def compile():

bin/upload-github

@@ -13,7 +13,7 @@ if [ "$1" == "" ]; then
     exit 1
 fi
 VERSION="$1"
-ANCHOR=$(grep '^## v' docs/content/changelog.md | head -n 1 | sed 's/^## //; s/[^A-Za-z0-9-]/-/g; s/--*/-/g')
+ANCHOR=$(grep '^## v' docs/content/changelog.md | head -1 | sed 's/^## //; s/[^A-Za-z0-9-]/-/g; s/--*/-/g')
 
 cat > "/tmp/${VERSION}-release-notes" <<EOF
 This is the ${VERSION} release of rclone.

cmd/bisync/bilib/output.go

@@ -5,13 +5,20 @@ import (
 	"bytes"
 	"log"
 
+	"github.com/rclone/rclone/fs"
 	"github.com/sirupsen/logrus"
 )
 
 // CaptureOutput runs a function capturing its output.
 func CaptureOutput(fun func()) []byte {
 	logSave := log.Writer()
-	logrusSave := logrus.StandardLogger().Out
+	logrusSave := logrus.StandardLogger().Writer()
+	defer func() {
+		err := logrusSave.Close()
+		if err != nil {
+			fs.Errorf(nil, "error closing logrusSave: %v", err)
+		}
+	}()
 	buf := &bytes.Buffer{}
 	log.SetOutput(buf)
 	logrus.SetOutput(buf)

cmd/bisync/bisync_test.go

@@ -746,16 +746,6 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 	case "test-func":
 		b.TestFn = testFunc
 		return
-	case "concurrent-func":
-		b.TestFn = func() {
-			src := filepath.Join(b.dataDir, "file7.txt")
-			dst := "file1.txt"
-			err := b.copyFile(ctx, src, b.replaceHex(b.path2), dst)
-			if err != nil {
-				fs.Errorf(src, "error copying file: %v", err)
-			}
-		}
-		return
 	case "fix-names":
 		// in case the local os converted any filenames
 		ci.NoUnicodeNormalization = true
@@ -881,9 +871,10 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 		if !ok || err != nil {
 			fs.Logf(remotePath, "Can't find expected file %s (was it renamed by the os?) %v", args[1], err)
 			return
+		} else {
+			// include hash of filename to make unicode form differences easier to see in logs
+			fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf))
 		}
-		// include hash of filename to make unicode form differences easier to see in logs
-		fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf))
 		return
 	default:
 		return fmt.Errorf("unknown command: %q", args[0])

cmd/bisync/compare.go

@@ -218,7 +218,7 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {
 	if b.opt.CompareFlag == "" {
 		return nil
 	}
-	var CompareFlag CompareOpt // for exclusions
+	var CompareFlag CompareOpt // for exlcusions
 	opts := strings.Split(b.opt.CompareFlag, ",")
 	for _, opt := range opts {
 		switch strings.ToLower(strings.TrimSpace(opt)) {

cmd/bisync/deltas.go

@@ -161,7 +161,9 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 		return
 	}
 
-	err = b.checkListing(now, newListing, "current "+msg)
+	if err == nil {
+		err = b.checkListing(now, newListing, "current "+msg)
+	}
 	if err != nil {
 		return
 	}
@@ -284,7 +286,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 }
 
 // applyDeltas
-func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (results2to1, results1to2 []Results, queues queues, err error) {
+func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changes1, changes2 bool, results2to1, results1to2 []Results, queues queues, err error) {
 	path1 := bilib.FsPath(b.fs1)
 	path2 := bilib.FsPath(b.fs2)
 
@@ -365,7 +367,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 		}
 	}
 
-	// if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
+	//if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
 	matches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
 
 	for _, file := range ds1.sort() {
@@ -390,7 +392,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 			} else if d2.is(deltaOther) {
 				b.indent("!WARNING", file, "New or changed in both paths")
 
-				// if files are identical, leave them alone instead of renaming
+				//if files are identical, leave them alone instead of renaming
 				if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {
 					fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
 					ls1.getPut(file, skippedDirs1)
@@ -484,6 +486,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 
 	// Do the batch operation
 	if copy2to1.NotEmpty() && !b.InGracefulShutdown {
+		changes1 = true
 		b.indent("Path2", "Path1", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 1)
 		results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")
@@ -495,11 +498,12 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 			return
 		}
 
-		// copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		//copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, &results2to1, "make")
 	}
 
 	if copy1to2.NotEmpty() && !b.InGracefulShutdown {
+		changes2 = true
 		b.indent("Path1", "Path2", "Do queued copies to")
 		ctx = b.setBackupDir(ctx, 2)
 		results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")
@@ -511,7 +515,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 			return
 		}
 
-		// copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		//copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, &results1to2, "make")
 	}
 
@@ -519,7 +523,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 		if err = b.saveQueue(delete1, "delete1"); err != nil {
 			return
 		}
-		// propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
+		//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, &results2to1, "remove")
 	}
 
@@ -527,7 +531,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 		if err = b.saveQueue(delete2, "delete2"); err != nil {
 			return
 		}
-		// propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
+		//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, &results1to2, "remove")
 	}
 

cmd/bisync/listing.go

@@ -394,7 +394,7 @@ func parseHash(str string) (string, string, error) {
 	return "", "", fmt.Errorf("invalid hash %q", str)
 }
 
-// checkListing verifies that listing is not empty (unless resyncing)
+// checkListing verifies that listing is not empty (unless resynching)
 func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error {
 	if b.opt.Resync || !ls.empty() {
 		return nil

cmd/bisync/log.go

@@ -66,8 +66,7 @@ func quotePath(path string) string {
 	return escapePath(path, true)
 }
 
-// Colors controls whether terminal colors are enabled
-var Colors bool
+var Colors bool // Colors controls whether terminal colors are enabled
 
 // Color handles terminal colors for bisync
 func Color(style string, s string) string {

cmd/bisync/operations.go

@@ -359,6 +359,8 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 
 	// Determine and apply changes to Path1 and Path2
 	noChanges := ds1.empty() && ds2.empty()
+	changes1 := false // 2to1
+	changes2 := false // 1to2
 	results2to1 := []Results{}
 	results1to2 := []Results{}
 
@@ -368,7 +370,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 		fs.Infof(nil, "No changes found")
 	} else {
 		fs.Infof(nil, "Applying changes")
-		results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
+		changes1, changes2, results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2)
 		if err != nil {
 			if b.InGracefulShutdown && (err == context.Canceled || err == accounting.ErrorMaxTransferLimitReachedGraceful || strings.Contains(err.Error(), "context canceled")) {
 				fs.Infof(nil, "Ignoring sync error due to Graceful Shutdown: %v", err)
@@ -393,11 +395,21 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
 	}
 	b.saveOldListings()
 	// save new listings
+	// NOTE: "changes" in this case does not mean this run vs. last run, it means start of this run vs. end of this run.
+	// i.e. whether we can use the March lst-new as this side's lst without modifying it.
 	if noChanges {
 		b.replaceCurrentListings()
 	} else {
-		err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) // 2to1
-		err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)  // 1to2
+		if changes1 || b.InGracefulShutdown { // 2to1
+			err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false)
+		} else {
+			err1 = bilib.CopyFileIfExists(b.newListing1, b.listing1)
+		}
+		if changes2 || b.InGracefulShutdown { // 1to2
+			err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true)
+		} else {
+			err2 = bilib.CopyFileIfExists(b.newListing2, b.listing2)
+		}
 	}
 	if b.DebugName != "" {
 		l1, _ := b.loadListing(b.listing1)

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -1 +0,0 @@
-"file1.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old

@@ -1,10 +0,0 @@
-# bisync listing v1 from test
--      109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
--       19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
--        0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"

cmd/bisync/testdata/test_concurrent/golden/test.log

@@ -1,73 +0,0 @@
-(01)  : test concurrent
-
-(02)  : test initial bisync
-(03)  : bisync resync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Copying Path2 files to Path1
-INFO  : - Path2    Resync is copying files to         - Path1
-INFO  : - Path1    Resync is copying files to         - Path2
-INFO  : Resync updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful
-
-(04)  : test changed on one path - file1
-(05)  : touch-glob 2001-01-02 {datadir/} file5R.txt
-(06)  : touch-glob 2023-08-26 {datadir/} file7.txt
-(07)  : copy-as {datadir/}file5R.txt {path2/} file1.txt
-
-(08)  : test bisync with file changed during
-(09)  : concurrent-func
-(10)  : bisync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Building Path1 and Path2 listings
-INFO  : Path1 checking for diffs
-INFO  : Path2 checking for diffs
-INFO  : - Path2    File changed: size (larger), time (newer) - file1.txt
-INFO  : Path2:    1 changes:    0 new,    1 modified,    0 deleted
-INFO  : (Modified:    1 newer,    0 older,    1 larger,    0 smaller)
-INFO  : Applying changes
-INFO  : - Path2    Queue copy to Path1       - {path1/}file1.txt
-INFO  : - Path2    Do queued copies to                - Path1
-INFO  : Updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful
-
-(11)  : bisync
-INFO  : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.
-INFO  : Bisyncing with Comparison Settings:
-{
-"Modtime": true,
-"Size": true,
-"Checksum": false,
-"NoSlowHash": false,
-"SlowHashSyncOnly": false,
-"DownloadHash": false
-}
-INFO  : Synching Path1 "{path1/}" with Path2 "{path2/}"
-INFO  : Building Path1 and Path2 listings
-INFO  : Path1 checking for diffs
-INFO  : Path2 checking for diffs
-INFO  : No changes found
-INFO  : Updating listings
-INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
-INFO  : Bisync successful

cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST

@@ -1 +0,0 @@
-This file is used for testing the health of rclone accesses to the local/remote file system.  Do not delete.

cmd/bisync/testdata/test_concurrent/modfiles/file1.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file10.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file11.txt

@@ -1 +0,0 @@
-This file is newer

cmd/bisync/testdata/test_concurrent/modfiles/file2.txt

@@ -1 +0,0 @@
-Newer version