Version v1.65.2

It was reported that rclone copy occasionally uploaded corrupted data
to azure blob.

This turned out to be a race condition updating the block count which
caused blocks to be duplicated.

This bug was introduced in this commit in v1.64.0 and will be fixed in v1.65.2

0427177857 azureblob: implement OpenChunkWriter and multi-thread uploads #7056

This race only seems to happen if `--checksum` is used but can happen otherwise.

Unfortunately Azure blob does not check the MD5 that we send them so
despite sending incorrect data this corruption is not detected. The
corruption is detected when rclone tries to download the file, so
attempting to copy the files back to local disk will result in errors
such as:

    ERROR : file.pokosuf5.partial: corrupted on transfer: md5 hash differ "XXX" vs "YYY"

This adds a check to test the blocklist we upload is as we expected
which would have caught the problem had it been in place earlier.

.gitattributes

@@ -1,7 +1,3 @@
-# Go writes go.mod and go.sum with lf even on windows
-go.mod text eol=lf
-go.sum text eol=lf
-
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true

.github/workflows/build.yml

@@ -17,21 +17,22 @@ on:
       manual:
         description: Manual run (bypass default conditions)
         type: boolean
+        required: true
         default: true
 
 jobs:
   build:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -42,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
+            os: macos-11
+            go: '1.21'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -57,15 +58,15 @@ jobs:
             deploy: true
 
           - job_name: mac_arm64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
+            os: macos-11
+            go: '1.21'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -75,20 +76,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.21
+          - job_name: go1.19
             os: ubuntu-latest
-            go: '1.21'
+            go: '1.19'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.22
+          - job_name: go1.20
             os: ubuntu-latest
-            go: '1.22'
+            go: '1.20'
             quicktest: true
             racequicktest: true
 
@@ -123,8 +124,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
@@ -137,8 +137,7 @@ jobs:
           brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
-          brew install git-annex git-annex-remote-rclone
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-11'
 
       - name: Install Libraries on Windows
         shell: powershell
@@ -168,6 +167,14 @@ jobs:
           printf "\n\nSystem environment:\n\n"
           env
 
+      - name: Go module cache
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Build rclone
         shell: bash
         run: |
@@ -217,77 +224,27 @@ jobs:
         if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "lint"
     runs-on: ubuntu-latest
 
     steps:
-      - name: Get runner parameters
-        id: get-runner-parameters
-        shell: bash
-        run: |
-          echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
-          echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
-
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Code quality test
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: latest
+
+      # Run govulncheck on the latest go version, the one we build binaries with
       - name: Install Go
-        id: setup-go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '1.21'
           check-latest: true
-          cache: false
-
-      - name: Cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            ~/.cache/golangci-lint
-          key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
-          restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
-
-      - name: Code quality test (Linux)
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (Windows)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "windows"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (macOS)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "darwin"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (FreeBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "freebsd"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (OpenBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "openbsd"
-        with:
-          version: latest
-          skip-cache: true
 
       - name: Install govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -296,7 +253,7 @@ jobs:
         run: govulncheck ./...
 
   android:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
+    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
     timeout-minutes: 30
     name: "android-all"
     runs-on: ubuntu-latest
@@ -311,7 +268,15 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '1.21'
+
+      - name: Go module cache
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_beta_docker_image.yml

@@ -0,0 +1,77 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                images: ghcr.io/${{ github.repository }}
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                registry: ghcr.io
+                # This is the user that triggered the Workflow. In this case, it will
+                # either be the user whom created the Release or manually triggered
+                # the workflow_dispatch.
+                username: ${{ github.actor }}
+                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+                # GitHub Actions at the start of a workflow run to identify the job.
+                # This is used to authenticate against GitHub Container Registry.
+                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+                # for more detailed information.
+                password: ${{ secrets.GITHUB_TOKEN }}
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .
+            - name: Build and publish image
+              uses: docker/build-push-action@v5
+              with:
+                file: Dockerfile
+                context: .
+                push: true # push the image to ghcr
+                tags: |
+                  ghcr.io/rclone/rclone:beta
+                  rclone/rclone:beta
+                labels: ${{ steps.meta.outputs.labels }}
+                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                cache-from: type=gha, scope=${{ github.workflow }}
+                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
+                provenance: false
+              # Eventually cache will need to be cleared if builds more frequent than once a week
+              # https://github.com/docker/build-push-action/issues/252
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .

.github/workflows/build_publish_docker_image.yml

@@ -1,294 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*-
-
-name: Build & Push Docker Images
-
-# Trigger the workflow on push or pull request
-on:
-  push:
-    branches:
-      - '**'
-    tags:
-      - '**'
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        default: true
-
-jobs:
-  build-image:
-    if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runs-on: ubuntu-24.04
-          - platform: linux/386
-            runs-on: ubuntu-24.04
-          - platform: linux/arm64
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v7
-            runs-on: ubuntu-24.04-arm
-          - platform: linux/arm/v6
-            runs-on: ubuntu-24.04-arm
-
-    name: Build Docker Image for ${{ matrix.platform }}
-    runs-on: ${{ matrix.runs-on }}
-
-    steps:
-      - name: Free Space
-        shell: bash
-        run: |
-          df -h .
-          # Remove android SDK
-          sudo rm -rf /usr/local/lib/android || true
-          # Remove .net runtime
-          sudo rm -rf /usr/share/dotnet || true
-          df -h .
-
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Set PLATFORM Variable
-        run: |
-          platform=${{ matrix.platform }}
-          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Set CACHE_NAME Variable
-        shell: python
-        run: |
-          import os, re
-
-          def slugify(input_string, max_length=63):
-            slug = input_string.lower()
-            slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
-            slug = slug.strip()
-            slug = re.sub(r'\s+', '-', slug)
-            slug = re.sub(r'-+', '-', slug)
-            slug = slug[:max_length]
-            slug = re.sub(r'[-]+$', '', slug)
-            return slug
-
-          ref_name_slug = "cache"
-
-          if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
-            ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"CACHE_NAME={ref_name_slug}\n")
-
-      - name: Get ImageOS
-        # There's no way around this, because "ImageOS" is only available to
-        # processes, but the setup-go action uses it in its key.
-        id: imageos
-        uses: actions/github-script@v7
-        with:
-          result-encoding: string
-          script: |
-            return process.env.ImageOS
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
-        with:
-          images: |
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Load Go Build Cache for Docker
-        id: go-cache
-        uses: actions/cache@v4
-        with:
-          key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          # Cache only the go builds, the module download is cached via the docker layer caching
-          path: |
-            go-build-cache
-
-      - name: Inject Go Build Cache into Docker
-        uses: reproducible-containers/buildkit-cache-dance@v3
-        with:
-          cache-map: |
-            {
-              "go-build-cache": "/root/.cache/go-build"
-            }
-          skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and Publish Image Digest
-        id: build
-        uses: docker/build-push-action@v6
-        with:
-          file: Dockerfile
-          context: .
-          provenance: false
-          # don't specify 'tags' here (error "get can't push tagged ref by digest")
-          # tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          annotations: ${{ steps.meta.outputs.annotations }}
-          platforms: ${{ matrix.platform }}
-          outputs: |
-            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
-          cache-to: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
-
-      - name: Export Image Digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload Image Digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM }}
-          path: /tmp/digests/*
-          retention-days: 1
-          if-no-files-found: error
-
-  merge-image:
-    name: Merge & Push Final Docker Image
-    runs-on: ubuntu-24.04
-    needs:
-      - build-image
-
-    steps:
-      - name: Download Image Digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digests-*
-          merge-multiple: true
-
-      - name: Set REPO_NAME Variable
-        run: |
-          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-
-      - name: Extract Metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
-        with:
-          images: |
-            ${{ env.REPO_NAME }}
-            ghcr.io/${{ env.REPO_NAME }}
-          labels: |
-            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-            org.opencontainers.image.vendor=${{ github.repository_owner }}
-            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          tags: |
-            type=sha
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=raw,value=beta,enable={{is_default_branch}}
-
-      - name: Extract Tags
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
-          tags_string = " ".join(tags)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"TAGS={tags_string}\n")
-
-      - name: Extract Annotations
-        shell: python
-        run: |
-          import json, os
-
-          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
-          metadata = json.loads(metadata_json)
-
-          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
-          annotations_string = " ".join(annotations)
-
-          with open(os.environ['GITHUB_ENV'], 'a') as env:
-              env.write(f"ANNOTATIONS={annotations_string}\n")
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          # This is the user that triggered the Workflow. In this case, it will
-          # either be the user whom created the Release or manually triggered
-          # the workflow_dispatch.
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create & Push Manifest List
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create \
-          ${{ env.TAGS }} \
-          ${{ env.ANNOTATIONS }} \
-          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
-
-      - name: Inspect and Run Multi-Platform Image
-        run: |
-          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
-          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_docker_plugin.yml

@@ -1,45 +0,0 @@
----
-# Github Actions release for rclone
-# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*-
-
-name: Release Build for Docker Plugin
-
-on:
-  release:
-    types: [published]
-
-jobs:
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,77 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        if: github.repository == 'rclone/rclone'
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@1.1.0
+              with:
+                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+    build_docker_volume_plugin:
+        if: github.repository == 'rclone/rclone'
+        needs: build
+        runs-on: ubuntu-latest
+        name: Build docker plugin job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Build and publish docker plugin
+              shell: bash
+              run: |
+                VER=${GITHUB_REF#refs/tags/}
+                PLUGIN_USER=rclone
+                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+                    export PLUGIN_USER PLUGIN_ARCH
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+                done
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/notify.yml

@@ -1,15 +0,0 @@
-name: Notify users based on issue labels
-
-on:
-  issues:
-      types: [labeled]
-
-jobs:
-  notify:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: jenschelkopf/issue-label-notification-action@1.3
-          with:
-             token: ${{ secrets.NOTIFY_ACTION_TOKEN }}
-             recipients: |
-                  Support Contract=@rclone/support

.github/workflows/winget.yml

@@ -1,14 +1,14 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.gitignore

@@ -3,13 +3,10 @@ _junk/
 rclone
 rclone.exe
 build
-/docs/public/
-/docs/.hugo_build.lock
-/docs/static/img/logos/
+docs/public
 rclone.iml
 .idea
 .history
-.vscode
 *.test
 *.iml
 fuzz-build.zip
@@ -18,5 +15,6 @@ fuzz-build.zip
 Thumbs.db
 __pycache__
 .DS_Store
+/docs/static/img/logos/
 resource_windows_*.syso
 .devcontainer

.golangci.yml

@@ -13,7 +13,6 @@ linters:
     - stylecheck
     - unused
     - misspell
-    - gocritic
     #- prealloc
     #- maligned
   disable-all: true
@@ -99,46 +98,3 @@ linters-settings:
     # Only enable the checks performed by the staticcheck stand-alone tool,
     # as documented here: https://staticcheck.io/docs/configuration/options/#checks
     checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
-  gocritic:
-    # Enable all default checks with some exceptions and some additions (commented).
-    # Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
-    disable-all: true
-    enabled-checks:
-      #- appendAssign # Enabled by default
-      - argOrder
-      - assignOp
-      - badCall
-      - badCond
-      #- captLocal # Enabled by default
-      - caseOrder
-      - codegenComment
-      #- commentFormatting # Enabled by default
-      - defaultCaseOrder
-      - deprecatedComment
-      - dupArg
-      - dupBranchBody
-      - dupCase
-      - dupSubExpr
-      - elseif
-      #- exitAfterDefer # Enabled by default
-      - flagDeref
-      - flagName
-      #- ifElseChain # Enabled by default
-      - mapKey
-      - newDeref
-      - offBy1
-      - regexpMust
-      - ruleguard # Not enabled by default
-      #- singleCaseSwitch # Enabled by default
-      - sloppyLen
-      - sloppyTypeAssert
-      - switchTrue
-      - typeSwitchVar
-      - underef
-      - unlambda
-      - unslice
-      - valSwap
-      - wrapperFunc
-    settings:
-      ruleguard:
-        rules: "${configDir}/bin/rules.go"

CONTRIBUTING.md

@@ -209,7 +209,7 @@ altogether with an HTML report and test retries then from the
 project root:
 
     go install github.com/rclone/rclone/fstest/test_all
-    test_all -backends drive
+    test_all -backend drive
 
 ### Full integration testing
 
@@ -490,7 +490,7 @@ alphabetical order of full name of remote (e.g. `drive` is ordered as
 - `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
     - make sure this has the `autogenerated options` comments in (see your reference backend docs)
     - update them in your backend with `bin/make_backend_docs.py remote`
-- `docs/content/overview.md` - overview docs - add an entry into the Features table and the Optional Features table.
+- `docs/content/overview.md` - overview docs
 - `docs/content/docs.md` - list of remotes in config section
 - `docs/content/_index.md` - front page of rclone.org
 - `docs/layouts/chrome/navbar.html` - add it to the website navigation
@@ -508,7 +508,7 @@ You'll need to modify the following files
 - `backend/s3/s3.go`
     - Add the provider to `providerOption` at the top of the file
     - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
-    - Exclude your provider from generic config questions (eg `region` and `endpoint).
+    - Exclude your provider from genric config questions (eg `region` and `endpoint).
     - Add the provider to the `setQuirks` function - see the documentation there.
 - `docs/content/s3.md`
     - Add the provider at the top of the page.

Dockerfile

@@ -1,47 +1,19 @@
 FROM golang:alpine AS builder
 
-ARG CGO_ENABLED=0
-
+COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN echo "**** Set Go Environment Variables ****" && \
-    go env -w GOCACHE=/root/.cache/go-build
-
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        make \
-        bash \
-        gawk \
-        git
-
-COPY go.mod .
-COPY go.sum .
-
-RUN echo "**** Download Go Dependencies ****" && \
-    go mod download -x
-
-RUN echo "**** Verify Go Dependencies ****" && \
-    go mod verify
-
-COPY . .
-
-RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
-    echo "**** Build Binary ****" && \
-    make
-
-RUN echo "**** Print Version Binary ****" && \
-    ./rclone version
+RUN apk add --no-cache make bash gawk git
+RUN \
+  CGO_ENABLED=0 \
+  make
+RUN ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN echo "**** Install Dependencies ****" && \
-    apk add --no-cache \
-        ca-certificates \
-        fuse3 \
-        tzdata && \
-    echo "Enable user_allow_other in fuse" && \
-    echo "user_allow_other" >> /etc/fuse.conf
+RUN apk --no-cache add ca-certificates fuse3 tzdata && \
+  echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

MAINTAINERS.md

@@ -21,8 +21,6 @@ Current active maintainers of rclone are:
 | Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
 | Hideo Aoyama     | @boukendesho      | snap packaging               |
 | nielash          | @nielash          | bisync                       |
-| Dan McArdle      | @dmcardle         | gitannex                     |
-| Sam Harrison     | @childish-sambino | filescom                     |
 
 **This is a work in progress Draft**
 

Makefile

@@ -36,14 +36,13 @@ ifdef BETA_SUBDIR
 endif
 BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
-BETA_UPLOAD_ROOT := beta.rclone.org:
+BETA_UPLOAD_ROOT := memstore:beta-rclone-org
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
 LINTTAGS=--build-tags "$(GOTAGS)"
 endif
-LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
 
 .PHONY: rclone test_all vars version
 
@@ -51,7 +50,7 @@ rclone:
 ifeq ($(GO_OS),windows)
 	go run bin/resource_windows.go -version $(TAG) -syso resource_windows_`go env GOARCH`.syso
 endif
-	go build -v $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS)
+	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
 ifeq ($(GO_OS),windows)
 	rm resource_windows_`go env GOARCH`.syso
 endif
@@ -60,7 +59,7 @@ endif
 	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
 
 test_all:
-	go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
+	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
 
 vars:
 	@echo SHELL="'$(SHELL)'"
@@ -88,13 +87,13 @@ test:	rclone test_all
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./...
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
 compiletest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
 
 # Do source code quality checks
 check:	rclone
@@ -104,7 +103,7 @@ check:	rclone
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -use-api -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
@@ -144,14 +143,10 @@ MANUAL.txt:	MANUAL.md
 	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
-	-@rmdir -p '$$HOME/.config/rclone'
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
-	@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
+	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
 
 backenddocs: rclone bin/make_backend_docs.py
-	-@rmdir -p '$$HOME/.config/rclone'
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
-	@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -172,7 +167,7 @@ website:
 	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
-	rclone -v sync docs/public www.rclone.org:
+	rclone -v sync docs/public memstore:www-rclone-org
 
 upload_test_website:	website
 	rclone -P sync docs/public test-rclone-org:
@@ -199,8 +194,8 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ downloads.rclone.org:/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
+	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
+	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -210,7 +205,7 @@ cross:	doc
 
 beta:
 	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone -v copy build/ pub.rclone.org:/$(TAG)
+	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
 log_since_last_release:
@@ -223,18 +218,18 @@ ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
@@ -243,7 +238,7 @@ fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server --logLevel info -w --disableFastRender
+	cd docs && hugo server -v -w --disableFastRender
 
 tag:	retag doc
 	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new

README.md

@@ -1,23 +1,7 @@
-<div align="center">
-<sup>Special thanks to our sponsor:</sup>
-<br>
-<br>
-<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
-  <div>
-    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
-  </div>
-  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
-  <div>
-    <sup>Visit warp.dev to learn more.</sup>
-  </div>
-</a>
-<br>
-<hr>
-</div>
-<br>
-
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
+[<img src="https://rclone.org/img/logos/warp-github-light.svg" title="Visit warp.dev to learn more." align="right">](https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103#gh-light-mode-only)
+[<img src="https://rclone.org/img/logos/warp-github-dark.svg" title="Visit warp.dev to learn more." align="right">](https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103#gh-dark-mode-only)
 
 [Website](https://rclone.org) |
 [Documentation](https://rclone.org/docs/) |
@@ -41,6 +25,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
   * Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
+  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
@@ -55,18 +40,14 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
-  * Files.com [:page_facing_up:](https://rclone.org/filescom/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
-  * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
-  * Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
   * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
-  * iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
   * ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
@@ -77,7 +58,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Linkbox [:page_facing_up:](https://rclone.org/linkbox)
   * Linode Object Storage [:page_facing_up:](https://rclone.org/s3/#linode)
-  * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
@@ -93,12 +73,10 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
-  * Outscale [:page_facing_up:](https://rclone.org/s3/#outscale)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
   * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
-  * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
@@ -107,11 +85,9 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
-  * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
-  * Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
@@ -119,7 +95,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
-  * Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)

RELEASE.md

@@ -37,44 +37,18 @@ This file describes how to make the various kinds of releases
 
 ## Update dependencies
 
-Early in the next release cycle update the dependencies.
+Early in the next release cycle update the dependencies
 
   * Review any pinned packages in go.mod and remove if possible
-  * `make updatedirect`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
-  * Fix anything which doesn't compile at this point and commit changes here
-  * `git commit -a -v -m "build: update all dependencies"`
-
-If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
-in the `go.mod`.
-
-```
-go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
-go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
-```
-
-If the `go mod tidy` fails use the output from it to remove the
-package which can't be upgraded from `/tmp/potential-upgrades` when
-done
-
-```
-git co go.mod go.sum
-```
-
-And try again.
-
-Optionally upgrade the direct and indirect dependencies. This is very
-likely to fail if the manual method was used abve - in that case
-ignore it as it is too time consuming to fix.
-
-  * `make update`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
+  * make updatedirect
+  * make GOTAGS=cmount
+  * make compiletest
+  * git commit -a -v
+  * make update
+  * make GOTAGS=cmount
+  * make compiletest
   * roll back any updates which didn't compile
-  * `git commit -a -v --amend`
+  * git commit -a -v --amend
   * **NB** watch out for this changing the default go version in `go.mod`
 
 Note that `make update` updates all direct and indirect dependencies
@@ -83,19 +57,6 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
-Once it compiles locally, push it on a test branch and commit fixes
-until the tests pass.
-
-### Major versions
-
-The above procedure will not upgrade major versions, so v2 to v3.
-However this tool can show which major versions might need to be
-upgraded:
-
-    go run github.com/icholy/gomajor@latest list -major
-
-Expect API breakage when updating major versions.
-
 ## Tidy beta
 
 At some point after the release run
@@ -178,8 +139,6 @@ docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/a
 
 To make a full build then set the tags correctly and add `--push`
 
-Note that you can't only build one architecture - you need to build them all.
-
 ```
-docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
+docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
 ```

VERSION

@@ -1 +1 @@
-v1.69.1
+v1.65.2

backend/alias/alias_internal_test.go

@@ -23,8 +23,8 @@ func prepare(t *testing.T, root string) {
 	configfile.Install()
 
 	// Configure the remote
-	config.FileSetValue(remoteName, "type", "alias")
-	config.FileSetValue(remoteName, "remote", root)
+	config.FileSet(remoteName, "type", "alias")
+	config.FileSet(remoteName, "remote", root)
 }
 
 func TestNewFS(t *testing.T) {
@@ -81,12 +81,10 @@ func TestNewFS(t *testing.T) {
 		for i, gotEntry := range gotEntries {
 			what := fmt.Sprintf("%s, entry=%d", what, i)
 			wantEntry := test.entries[i]
-			_, isDir := gotEntry.(fs.Directory)
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			if !isDir {
-				require.Equal(t, wantEntry.size, gotEntry.Size(), what)
-			}
+			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
 	}

backend/all/all.go

@@ -4,13 +4,13 @@ package all
 import (
 	// Active file systems
 	_ "github.com/rclone/rclone/backend/alias"
+	_ "github.com/rclone/rclone/backend/amazonclouddrive"
 	_ "github.com/rclone/rclone/backend/azureblob"
 	_ "github.com/rclone/rclone/backend/azurefiles"
 	_ "github.com/rclone/rclone/backend/b2"
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
 	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/cloudinary"
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
@@ -18,16 +18,13 @@ import (
 	_ "github.com/rclone/rclone/backend/dropbox"
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/filescom"
 	_ "github.com/rclone/rclone/backend/ftp"
-	_ "github.com/rclone/rclone/backend/gofile"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
 	_ "github.com/rclone/rclone/backend/hdfs"
 	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/iclouddrive"
 	_ "github.com/rclone/rclone/backend/imagekit"
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
@@ -43,7 +40,6 @@ import (
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/pikpak"
-	_ "github.com/rclone/rclone/backend/pixeldrain"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
@@ -58,7 +54,6 @@ import (
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
-	_ "github.com/rclone/rclone/backend/ulozto"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -0,0 +1,21 @@
+// Test AmazonCloudDrive filesystem interface
+
+//go:build acd
+// +build acd
+
+package amazonclouddrive_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/amazonclouddrive"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
+	fstests.RemoteName = "TestAmazonCloudDrive:"
+	fstests.Run(t)
+}

backend/azureblob/azureblob.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
@@ -209,22 +210,6 @@ rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
 keys instead of setting ` + "`service_principal_file`" + `.
 `,
 			Advanced: true,
-		}, {
-			Name: "disable_instance_discovery",
-			Help: `Skip requesting Microsoft Entra instance metadata
-
-This should be set true only by applications authenticating in
-disconnected clouds, or private clouds such as Azure Stack.
-
-It determines whether rclone requests Microsoft Entra instance
-metadata from ` + "`https://login.microsoft.com/`" + ` before
-authenticating.
-
-Setting this to true will skip this request, making you responsible
-for ensuring the configured authority is valid and trustworthy.
-`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name: "use_msi",
 			Help: `Use a managed service identity to authenticate (only works in Azure).
@@ -259,20 +244,6 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Help:     "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "use_az",
-			Help: `Use Azure CLI tool az for authentication
-
-Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
-as the sole means of authentication.
-
-Setting this can be useful if you wish to use the az CLI on a host with
-a System Managed Identity that you do not want to use.
-
-Don't set env_auth at the same time.
-`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -431,24 +402,6 @@ rclone does if you know the container exists already.
 			Help:     `If set, do not do HEAD before GET when getting objects.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "delete_snapshots",
-			Help: `Set to specify how to deal with snapshots on blob deletion.`,
-			Examples: []fs.OptionExample{
-				{
-					Value: "",
-					Help:  "By default, the delete operation fails if a blob has snapshots",
-				}, {
-					Value: string(blob.DeleteSnapshotsOptionTypeInclude),
-					Help:  "Specify 'include' to remove the root blob and all its snapshots",
-				}, {
-					Value: string(blob.DeleteSnapshotsOptionTypeOnly),
-					Help:  "Specify 'only' to remove only the snapshots but keep the root blob.",
-				},
-			},
-			Default:   "",
-			Exclusive: true,
-			Advanced:  true,
 		}},
 	})
 }
@@ -468,12 +421,10 @@ type Options struct {
 	Username                   string               `config:"username"`
 	Password                   string               `config:"password"`
 	ServicePrincipalFile       string               `config:"service_principal_file"`
-	DisableInstanceDiscovery   bool                 `config:"disable_instance_discovery"`
 	UseMSI                     bool                 `config:"use_msi"`
 	MSIObjectID                string               `config:"msi_object_id"`
 	MSIClientID                string               `config:"msi_client_id"`
 	MSIResourceID              string               `config:"msi_mi_res_id"`
-	UseAZ                      bool                 `config:"use_az"`
 	Endpoint                   string               `config:"endpoint"`
 	ChunkSize                  fs.SizeSuffix        `config:"chunk_size"`
 	UploadConcurrency          int                  `config:"upload_concurrency"`
@@ -487,7 +438,6 @@ type Options struct {
 	DirectoryMarkers           bool                 `config:"directory_markers"`
 	NoCheckContainer           bool                 `config:"no_check_container"`
 	NoHeadObject               bool                 `config:"no_head_object"`
-	DeleteSnapshots            string               `config:"delete_snapshots"`
 }
 
 // Fs represents a remote azure server
@@ -743,11 +693,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ClientOptions: policyClientOptions,
 	}
 
-	// Here we auth by setting one of cred, sharedKeyCred, f.svc or anonymous
+	// Here we auth by setting one of cred, sharedKeyCred or f.svc
 	var (
 		cred          azcore.TokenCredential
 		sharedKeyCred *service.SharedKeyCredential
-		anonymous     = false
 	)
 	switch {
 	case opt.EnvAuth:
@@ -757,8 +706,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 		// Read credentials from the environment
 		options := azidentity.DefaultAzureCredentialOptions{
-			ClientOptions:            policyClientOptions,
-			DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
+			ClientOptions: policyClientOptions,
 		}
 		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
@@ -908,15 +856,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
-	case opt.UseAZ:
-		var options = azidentity.AzureCLICredentialOptions{}
-		cred, err = azidentity.NewAzureCLICredential(&options)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
-		}
-	case opt.Account != "":
-		// Anonymous access
-		anonymous = true
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -946,12 +885,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			if err != nil {
 				return nil, fmt.Errorf("create client failed: %w", err)
 			}
-		} else if anonymous {
-			// Anonymous public access
-			f.svc, err = service.NewClientWithNoCredential(opt.Endpoint, &clientOpt)
-			if err != nil {
-				return nil, fmt.Errorf("create public client failed: %w", err)
-			}
 		}
 	}
 	if f.svc == nil {
@@ -1137,7 +1070,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 			isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == f.opt.Enc.ToStandardPath(directory) {
+				if remote == directory {
 					continue
 				}
 				// process directory markers as directories
@@ -2133,6 +2066,7 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		return 0, nil
 	}
 	md5sum := m.Sum(nil)
+	transactionalMD5 := md5sum[:]
 
 	// increment the blockID and save the blocks for finalize
 	var binaryBlockID [8]byte // block counter as LSB first 8 bytes
@@ -2155,16 +2089,13 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		}
 		options := blockblob.StageBlockOptions{
 			// Specify the transactional md5 for the body, to be validated by the service.
-			TransactionalValidation: blob.TransferValidationTypeMD5(md5sum),
+			TransactionalValidation: blob.TransferValidationTypeMD5(transactionalMD5),
 		}
 		_, err = w.ui.blb.StageBlock(ctx, blockID, &readSeekCloser{Reader: reader, Seeker: reader}, &options)
 		if err != nil {
 			if chunkNumber <= 8 {
 				return w.f.shouldRetry(ctx, err)
 			}
-			if fserrors.ContextError(ctx, &err) {
-				return false, err
-			}
 			// retry all chunks once have done the first few
 			return true, err
 		}
@@ -2425,10 +2356,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	blb := o.getBlobSVC()
-	opt := blob.DeleteOptions{}
-	if o.fs.opt.DeleteSnapshots != "" {
-		action := blob.DeleteSnapshotsOptionType(o.fs.opt.DeleteSnapshots)
-		opt.DeleteSnapshots = &action
+	//only := blob.DeleteSnapshotsOptionTypeOnly
+	opt := blob.DeleteOptions{
+		//DeleteSnapshots: &only,
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blb.Delete(ctx, &opt)

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,7 @@
 // Test AzureBlob filesystem interface
 
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || solaris || js
+// +build plan9 solaris js
 
-// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob

backend/azurefiles/azurefiles.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 // Package azurefiles provides an interface to Microsoft Azure Files
 package azurefiles
@@ -393,10 +394,8 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 	policyClientOptions := policy.ClientOptions{
 		Transport: newTransporter(ctx),
 	}
-	backup := service.ShareTokenIntentBackup
 	clientOpt := service.ClientOptions{
-		ClientOptions:     policyClientOptions,
-		FileRequestIntent: &backup,
+		ClientOptions: policyClientOptions,
 	}
 
 	// Here we auth by setting one of cred, sharedKeyCred or f.client
@@ -899,7 +898,7 @@ func (o *Object) getMetadata(ctx context.Context) error {
 
 // Hash returns the MD5 of an object returning a lowercase hex string
 //
-// May make a network request because the [fs.List] method does not
+// May make a network request becaue the [fs.List] method does not
 // return MD5 hashes for DirEntry
 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
 	if ty != hash.MD5 {
@@ -1037,10 +1036,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if _, createErr := fc.Create(ctx, size, nil); createErr != nil {
 			return fmt.Errorf("update: unable to create file: %w", createErr)
 		}
-	} else if size != o.Size() {
+	} else {
 		// Resize the file if needed
-		if _, resizeErr := fc.Resize(ctx, size, nil); resizeErr != nil {
-			return fmt.Errorf("update: unable to resize while trying to update: %w ", resizeErr)
+		if size != o.Size() {
+			if _, resizeErr := fc.Resize(ctx, size, nil); resizeErr != nil {
+				return fmt.Errorf("update: unable to resize while trying to update: %w ", resizeErr)
+			}
 		}
 	}
 

backend/azurefiles/azurefiles_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package azurefiles
 

backend/azurefiles/azurefiles_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package azurefiles
 

backend/azurefiles/azurefiles_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || js
+// +build plan9 js
 
-// Package azurefiles provides an interface to Microsoft Azure Files
 package azurefiles

backend/b2/api/types.go

@@ -42,10 +42,9 @@ type Bucket struct {
 
 // LifecycleRule is a single lifecycle rule
 type LifecycleRule struct {
-	DaysFromHidingToDeleting                        *int   `json:"daysFromHidingToDeleting"`
-	DaysFromUploadingToHiding                       *int   `json:"daysFromUploadingToHiding"`
-	DaysFromStartingToCancelingUnfinishedLargeFiles *int   `json:"daysFromStartingToCancelingUnfinishedLargeFiles"`
-	FileNamePrefix                                  string `json:"fileNamePrefix"`
+	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
+	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
+	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base

backend/b2/api/types_test.go

@@ -42,11 +42,11 @@ func TestTimestampIsZero(t *testing.T) {
 }
 
 func TestTimestampEqual(t *testing.T) {
-	assert.False(t, emptyT.Equal(emptyT)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.False(t, emptyT.Equal(emptyT))
 	assert.False(t, t0.Equal(emptyT))
 	assert.False(t, emptyT.Equal(t0))
 	assert.False(t, t0.Equal(t1))
 	assert.False(t, t1.Equal(t0))
-	assert.True(t, t0.Equal(t0)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
-	assert.True(t, t1.Equal(t1)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.True(t, t0.Equal(t0))
+	assert.True(t, t1.Equal(t1))
 }

backend/b2/b2.go

@@ -60,7 +60,6 @@ const (
 	defaultChunkSize    = 96 * fs.Mebi
 	defaultUploadCutoff = 200 * fs.Mebi
 	largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
-	defaultMaxAge       = 24 * time.Hour
 )
 
 // Globals
@@ -102,7 +101,7 @@ below will cause b2 to return specific errors:
   * "force_cap_exceeded"
 
 These will be set in the "X-Bz-Test-Mode" header which is documented
-in the [b2 integrations checklist](https://www.backblaze.com/docs/cloud-storage-integration-checklist).`,
+in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).`,
 			Default:  "",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
@@ -194,12 +193,9 @@ Example:
 			Advanced: true,
 		}, {
 			Name: "download_auth_duration",
-			Help: `Time before the public link authorization token will expire in s or suffix ms|s|m|h|d.
-
-This is used in combination with "rclone link" for making files
-accessible to the public and sets the duration before the download
-authorization token will expire.
+			Help: `Time before the authorization token will expire in s or suffix ms|s|m|h|d.
 
+The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
@@ -244,7 +240,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			// See: https://www.backblaze.com/docs/cloud-storage-files
+			// See: https://www.backblaze.com/b2/docs/files.html
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// FIXME: allow /, but not leading, trailing or double
 			Default: (encoder.Display |
@@ -363,7 +359,7 @@ var retryErrorCodes = []int{
 	504, // Gateway Time-out
 }
 
-// shouldRetryNoReauth returns a boolean as to whether this resp and err
+// shouldRetryNoAuth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
@@ -1249,7 +1245,7 @@ func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 // if oldOnly is true then it deletes only non current files.
 //
 // Implemented here so we can make sure we delete old versions.
-func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) error {
+func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 	bucket, directory := f.split(dir)
 	if bucket == "" {
 		return errors.New("can't purge from root")
@@ -1267,7 +1263,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 		}
 	}
 	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		return time.Since(time.Time(timestamp)) > maxAge
+		return time.Since(time.Time(timestamp)).Hours() > 24
 	}
 
 	// Delete Config.Transfers in parallel
@@ -1290,21 +1286,6 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 			}
 		}()
 	}
-	if oldOnly {
-		if deleteHidden && deleteUnfinished {
-			fs.Infof(f, "cleaning bucket %q of all hidden files, and pending multipart uploads older than %v", bucket, maxAge)
-		} else if deleteHidden {
-			fs.Infof(f, "cleaning bucket %q of all hidden files", bucket)
-		} else if deleteUnfinished {
-			fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than %v", bucket, maxAge)
-		} else {
-			fs.Errorf(f, "cleaning bucket %q of nothing. This should never happen!", bucket)
-			return nil
-		}
-	} else {
-		fs.Infof(f, "cleaning bucket %q of all files", bucket)
-	}
-
 	last := ""
 	checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
 		if !isDirectory {
@@ -1315,14 +1296,14 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 			if oldOnly && last != remote {
 				// Check current version of the file
-				if deleteHidden && object.Action == "hide" {
+				if object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
 					toBeDeleted <- object
-				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
+				} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
 					toBeDeleted <- object
 				} else {
-					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
+					fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
@@ -1344,17 +1325,12 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 
 // Purge deletes all the files and directories including the old versions.
 func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purge(ctx, dir, false, false, false, defaultMaxAge)
+	return f.purge(ctx, dir, false)
 }
 
-// CleanUp deletes all hidden files and pending multipart uploads older than 24 hours.
+// CleanUp deletes all the hidden files.
 func (f *Fs) CleanUp(ctx context.Context) error {
-	return f.purge(ctx, "", true, true, true, defaultMaxAge)
-}
-
-// cleanUp deletes all hidden files and/or pending multipart uploads older than the specified age.
-func (f *Fs) cleanUp(ctx context.Context, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) (err error) {
-	return f.purge(ctx, "", true, deleteHidden, deleteUnfinished, maxAge)
+	return f.purge(ctx, "", true)
 }
 
 // copy does a server-side copy from dstObj <- srcObj
@@ -1566,7 +1542,7 @@ func (o *Object) Size() int64 {
 //
 // Make sure it is lower case.
 //
-// Remove unverified prefix - see https://www.backblaze.com/docs/cloud-storage-upload-files-with-the-native-api
+// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
 // Some tools (e.g. Cyberduck) use this
 func cleanSHA1(sha1 string) string {
 	const unverified = "unverified:"
@@ -1593,11 +1569,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
-	err = o.parseTimeString(Info[timeKey])
-	if err != nil {
-		return err
-	}
-	return nil
+	return o.parseTimeString(Info[timeKey])
 }
 
 // decodeMetaData sets the metadata in the object from an api.File
@@ -1699,16 +1671,6 @@ func timeString(modTime time.Time) string {
 	return strconv.FormatInt(modTime.UnixNano()/1e6, 10)
 }
 
-// parseTimeStringHelper converts a decimal string number of milliseconds
-// elapsed since January 1, 1970 UTC into a time.Time
-func parseTimeStringHelper(timeString string) (time.Time, error) {
-	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
-	if err != nil {
-		return time.Time{}, err
-	}
-	return time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC(), nil
-}
-
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1716,12 +1678,12 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	if timeString == "" {
 		return nil
 	}
-	modTime, err := parseTimeStringHelper(timeString)
+	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
 		return nil
 	}
-	o.modTime = modTime
+	o.modTime = time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC()
 	return nil
 }
 
@@ -1798,14 +1760,14 @@ func (file *openFile) Close() (err error) {
 
 	// Check to see we read the correct number of bytes
 	if file.o.Size() != file.bytes {
-		return fmt.Errorf("corrupted on transfer: lengths differ want %d vs got %d", file.o.Size(), file.bytes)
+		return fmt.Errorf("object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes)
 	}
 
 	// Check the SHA1
 	receivedSHA1 := file.o.sha1
 	calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil))
 	if receivedSHA1 != "" && receivedSHA1 != calculatedSHA1 {
-		return fmt.Errorf("corrupted on transfer: SHA1 hashes differ want %q vs got %q", receivedSHA1, calculatedSHA1)
+		return fmt.Errorf("object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1)
 	}
 
 	return nil
@@ -1875,7 +1837,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		ContentType:     resp.Header.Get("Content-Type"),
 		Info:            Info,
 	}
-
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -1973,7 +1934,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil, options...)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
 				o.fs.putRW(rw)
 				return err
@@ -2005,10 +1966,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return o.decodeMetaDataFileInfo(up.info)
 	}
 
-	modTime, err := o.getModTime(ctx, src, options)
-	if err != nil {
-		return err
-	}
+	modTime := src.ModTime(ctx)
 
 	calculatedSha1, _ := src.Hash(ctx, hash.SHA1)
 	if calculatedSha1 == "" {
@@ -2113,36 +2071,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.decodeMetaDataFileInfo(&response)
 }
 
-// Get modTime from the source; if --metadata is set, fetch the src metadata and get it from there.
-// When metadata support is added to b2, this method will need a more generic name
-func (o *Object) getModTime(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (time.Time, error) {
-	modTime := src.ModTime(ctx)
-
-	// Fetch metadata if --metadata is in use
-	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
-	if err != nil {
-		return time.Time{}, fmt.Errorf("failed to read metadata from source object: %w", err)
-	}
-	// merge metadata into request and user metadata
-	for k, v := range meta {
-		k = strings.ToLower(k)
-		// For now, the only metadata we're concerned with is "mtime"
-		switch k {
-		case "mtime":
-			// mtime in meta overrides source ModTime
-			metaModTime, err := time.Parse(time.RFC3339Nano, v)
-			if err != nil {
-				fs.Debugf(o, "failed to parse metadata %s: %q: %v", k, v, err)
-			} else {
-				modTime = metaModTime
-			}
-		default:
-			// Do nothing for now
-		}
-	}
-	return modTime, nil
-}
-
 // OpenChunkWriter returns the chunk size and a ChunkWriter
 //
 // Pass in the remote and the src object
@@ -2174,7 +2102,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 		Concurrency: o.fs.opt.UploadConcurrency,
 		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
 	}
-	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
+	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil)
 	return info, up, err
 }
 
@@ -2220,7 +2148,6 @@ This will dump something like this showing the lifecycle rules.
         {
             "daysFromHidingToDeleting": 1,
             "daysFromUploadingToHiding": null,
-            "daysFromStartingToCancelingUnfinishedLargeFiles": null,
             "fileNamePrefix": ""
         }
     ]
@@ -2247,9 +2174,8 @@ overwrites will still cause versions to be made.
 See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
 `,
 	Opts: map[string]string{
-		"daysFromHidingToDeleting":                        "After a file has been hidden for this many days it is deleted. 0 is off.",
-		"daysFromUploadingToHiding":                       "This many days after uploading a file is hidden",
-		"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
+		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
+		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
 	},
 }
 
@@ -2269,13 +2195,6 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 		}
 		newRule.DaysFromUploadingToHiding = &days
 	}
-	if daysStr := opt["daysFromStartingToCancelingUnfinishedLargeFiles"]; daysStr != "" {
-		days, err := strconv.Atoi(daysStr)
-		if err != nil {
-			return nil, fmt.Errorf("bad daysFromStartingToCancelingUnfinishedLargeFiles: %w", err)
-		}
-		newRule.DaysFromStartingToCancelingUnfinishedLargeFiles = &days
-	}
 	bucketName, _ := f.split("")
 	if bucketName == "" {
 		return nil, errors.New("bucket required")
@@ -2283,7 +2202,7 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 	}
 
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err
@@ -2321,56 +2240,8 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 	return bucket.LifecycleRules, nil
 }
 
-var cleanupHelp = fs.CommandHelp{
-	Name:  "cleanup",
-	Short: "Remove unfinished large file uploads.",
-	Long: `This command removes unfinished large file uploads of age greater than
-max-age, which defaults to 24 hours.
-
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
-
-    rclone backend cleanup b2:bucket/path/to/object
-    rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
-
-Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
-`,
-	Opts: map[string]string{
-		"max-age": "Max age of upload to delete",
-	},
-}
-
-func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	maxAge := defaultMaxAge
-	if opt["max-age"] != "" {
-		maxAge, err = fs.ParseDuration(opt["max-age"])
-		if err != nil {
-			return nil, fmt.Errorf("bad max-age: %w", err)
-		}
-	}
-	return nil, f.cleanUp(ctx, false, true, maxAge)
-}
-
-var cleanupHiddenHelp = fs.CommandHelp{
-	Name:  "cleanup-hidden",
-	Short: "Remove old versions of files.",
-	Long: `This command removes any old hidden versions of files.
-
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
-
-    rclone backend cleanup-hidden b2:bucket/path/to/dir
-`,
-}
-
-func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	return nil, f.cleanUp(ctx, true, false, 0)
-}
-
 var commandHelp = []fs.CommandHelp{
 	lifecycleHelp,
-	cleanupHelp,
-	cleanupHiddenHelp,
 }
 
 // Command the backend to run a named command
@@ -2386,10 +2257,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	switch name {
 	case "lifecycle":
 		return f.lifecycleCommand(ctx, name, arg, opt)
-	case "cleanup":
-		return f.cleanupCommand(ctx, name, arg, opt)
-	case "cleanup-hidden":
-		return f.cleanupHiddenCommand(ctx, name, arg, opt)
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}

backend/b2/b2_internal_test.go

@@ -1,29 +1,15 @@
 package b2
 
 import (
-	"context"
-	"crypto/sha1"
-	"fmt"
-	"path"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/bucket"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/lib/version"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // Test b2 string encoding
-// https://www.backblaze.com/docs/cloud-storage-native-api-string-encoding
+// https://www.backblaze.com/b2/docs/string_encoding.html
 
 var encodeTest = []struct {
 	fullyEncoded     string
@@ -184,297 +170,9 @@ func TestParseTimeString(t *testing.T) {
 
 }
 
-// Return a map of the headers in the options with keys stripped of the "x-bz-info-" prefix
-func OpenOptionToMetaData(options []fs.OpenOption) map[string]string {
-	var headers = make(map[string]string)
-	for _, option := range options {
-		k, v := option.Header()
-		k = strings.ToLower(k)
-		if strings.HasPrefix(k, headerPrefix) {
-			headers[k[len(headerPrefix):]] = v
-		}
-	}
-
-	return headers
-}
-
-func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string, chunkSize string) {
-	what := fmt.Sprintf("Size%s/UploadCutoff%s/ChunkSize%s", size, uploadCutoff, chunkSize)
-	t.Run(what, func(t *testing.T) {
-		ctx := context.Background()
-
-		ss := fs.SizeSuffix(0)
-		err := ss.Set(size)
-		require.NoError(t, err)
-		original := random.String(int(ss))
-
-		contents := fstest.Gz(t, original)
-		mimeType := "text/html"
-
-		if chunkSize != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(chunkSize)
-			require.NoError(t, err)
-			_, err = f.SetUploadChunkSize(ss)
-			require.NoError(t, err)
-		}
-
-		if uploadCutoff != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(uploadCutoff)
-			require.NoError(t, err)
-			_, err = f.SetUploadCutoff(ss)
-			require.NoError(t, err)
-		}
-
-		item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
-		btime := time.Now()
-		metadata := fs.Metadata{
-			// Just mtime for now - limit to milliseconds since x-bz-info-src_last_modified_millis can't support any
-
-			"mtime": "2009-05-06T04:05:06.499Z",
-		}
-
-		// Need to specify HTTP options with the header prefix since they are passed as-is
-		options := []fs.OpenOption{
-			&fs.HTTPOption{Key: "X-Bz-Info-a", Value: "1"},
-			&fs.HTTPOption{Key: "X-Bz-Info-b", Value: "2"},
-		}
-
-		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, mimeType, metadata, options...)
-		defer func() {
-			assert.NoError(t, obj.Remove(ctx))
-		}()
-		o := obj.(*Object)
-		gotMetadata, err := o.getMetaData(ctx)
-		require.NoError(t, err)
-
-		// X-Bz-Info-a & X-Bz-Info-b
-		optMetadata := OpenOptionToMetaData(options)
-		for k, v := range optMetadata {
-			got := gotMetadata.Info[k]
-			assert.Equal(t, v, got, k)
-		}
-
-		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
-
-		// Modification time from the x-bz-info-src_last_modified_millis header
-		var mtime api.Timestamp
-		err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
-		if err != nil {
-			fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
-		}
-		assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
-
-		// Upload time
-		gotBtime := time.Time(gotMetadata.UploadTimestamp)
-		dt := gotBtime.Sub(btime)
-		assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
-
-		t.Run("GzipEncoding", func(t *testing.T) {
-			// Test that the gzipped file we uploaded can be
-			// downloaded
-			checkDownload := func(wantContents string, wantSize int64, wantHash string) {
-				gotContents := fstests.ReadObject(ctx, t, o, -1)
-				assert.Equal(t, wantContents, gotContents)
-				assert.Equal(t, wantSize, o.Size())
-				gotHash, err := o.Hash(ctx, hash.SHA1)
-				require.NoError(t, err)
-				assert.Equal(t, wantHash, gotHash)
-			}
-
-			t.Run("NoDecompress", func(t *testing.T) {
-				checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
-			})
-		})
-	})
-}
-
-func (f *Fs) InternalTestMetadata(t *testing.T) {
-	// 1 kB regular file
-	f.internalTestMetadata(t, "1kiB", "", "")
-
-	// 10 MiB large file
-	f.internalTestMetadata(t, "10MiB", "6MiB", "6MiB")
-}
-
-func sha1Sum(t *testing.T, s string) string {
-	hash := sha1.Sum([]byte(s))
-	return fmt.Sprintf("%x", hash)
-}
-
-// This is adapted from the s3 equivalent.
-func (f *Fs) InternalTestVersions(t *testing.T) {
-	ctx := context.Background()
-
-	// Small pause to make the LastModified different since AWS
-	// only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// Create an object
-	const dirName = "versions"
-	const fileName = dirName + "/" + "test-versions.txt"
-	contents := random.String(100)
-	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
-	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-	defer func() {
-		assert.NoError(t, obj.Remove(ctx))
-	}()
-	objMetadata, err := obj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	// Small pause
-	time.Sleep(2 * time.Second)
-
-	// Remove it
-	assert.NoError(t, obj.Remove(ctx))
-
-	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// And create it with different size and contents
-	newContents := random.String(101)
-	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
-	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
-	newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	t.Run("Versions", func(t *testing.T) {
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-
-		// Read the contents
-		entries, err := f.List(ctx, dirName)
-		require.NoError(t, err)
-		tests := 0
-		var fileNameVersion string
-		for _, entry := range entries {
-			t.Log(entry)
-			remote := entry.Remote()
-			if remote == fileName {
-				t.Run("ReadCurrent", func(t *testing.T) {
-					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				tests++
-			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
-				t.Run("ReadVersion", func(t *testing.T) {
-					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
-				fileNameVersion = remote
-				tests++
-			}
-		}
-		assert.Equal(t, 2, tests, "object missing from listing")
-
-		// Check we can read the object with a version suffix
-		t.Run("NewObject", func(t *testing.T) {
-			o, err := f.NewObject(ctx, fileNameVersion)
-			require.NoError(t, err)
-			require.NotNil(t, o)
-			assert.Equal(t, int64(100), o.Size(), o.Remote())
-		})
-
-		// Check we can make a NewFs from that object with a version suffix
-		t.Run("NewFs", func(t *testing.T) {
-			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
-			// Make sure --b2-versions is set in the config of the new remote
-			fs.Debugf(nil, "oldPath = %q", newPath)
-			lastColon := strings.LastIndex(newPath, ":")
-			require.True(t, lastColon >= 0)
-			newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
-			fs.Debugf(nil, "newPath = %q", newPath)
-			fNew, err := cache.Get(ctx, newPath)
-			// This should return pointing to a file
-			require.Equal(t, fs.ErrorIsFile, err)
-			require.NotNil(t, fNew)
-			// With the directory above
-			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
-		})
-	})
-
-	t.Run("VersionAt", func(t *testing.T) {
-		// We set --b2-version-at for this test so make sure we reset it at the end
-		defer func() {
-			f.opt.VersionAt = fs.Time{}
-		}()
-
-		var (
-			firstObjectTime  = time.Time(objMetadata.UploadTimestamp)
-			secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
-		)
-
-		for _, test := range []struct {
-			what     string
-			at       time.Time
-			want     []fstest.Item
-			wantErr  error
-			wantSize int64
-		}{
-			{
-				what:    "Before",
-				at:      firstObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterOne",
-				at:       firstObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
-				wantSize: 100,
-			},
-			{
-				what:    "AfterDelete",
-				at:      secondObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterTwo",
-				at:       secondObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
-				wantSize: 101,
-			},
-		} {
-			t.Run(test.what, func(t *testing.T) {
-				f.opt.VersionAt = fs.Time(test.at)
-				t.Run("List", func(t *testing.T) {
-					fstest.CheckListing(t, f, test.want)
-				})
-				// b2 NewObject doesn't work with VersionAt
-				//t.Run("NewObject", func(t *testing.T) {
-				//	gotObj, gotErr := f.NewObject(ctx, fileName)
-				//	assert.Equal(t, test.wantErr, gotErr)
-				//	if gotErr == nil {
-				//		assert.Equal(t, test.wantSize, gotObj.Size())
-				//	}
-				//})
-			})
-		}
-	})
-
-	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
-	})
-
-	// Purge gets tested later
-}
-
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Metadata", f.InternalTestMetadata)
-	t.Run("Versions", f.InternalTestVersions)
+	// Internal tests go here
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/upload.go

@@ -1,6 +1,6 @@
 // Upload large files for b2
 //
-// Docs - https://www.backblaze.com/docs/cloud-storage-large-files
+// Docs - https://www.backblaze.com/b2/docs/large_files.html
 
 package b2
 
@@ -91,7 +91,7 @@ type largeUpload struct {
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File, options ...fs.OpenOption) (up *largeUpload, err error) {
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
 	parts := 0
 	chunkSize := defaultChunkSize
@@ -104,6 +104,11 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 			parts++
 		}
 	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_start_large_file",
+	}
 	bucket, bucketPath := o.split()
 	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
@@ -113,27 +118,12 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		BucketID: bucketID,
 		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
-	optionsToSend := make([]fs.OpenOption, 0, len(options))
 	if newInfo == nil {
-		modTime, err := o.getModTime(ctx, src, options)
-		if err != nil {
-			return nil, err
-		}
-
+		modTime := src.ModTime(ctx)
 		request.ContentType = fs.MimeType(ctx, src)
 		request.Info = map[string]string{
 			timeKey: timeString(modTime),
 		}
-		// Custom upload headers - remove header prefix since they are sent in the body
-		for _, option := range options {
-			k, v := option.Header()
-			k = strings.ToLower(k)
-			if strings.HasPrefix(k, headerPrefix) {
-				request.Info[k[len(headerPrefix):]] = v
-			} else {
-				optionsToSend = append(optionsToSend, option)
-			}
-		}
 		// Set the SHA1 if known
 		if !o.fs.opt.DisableCheckSum || doCopy {
 			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
@@ -144,11 +134,6 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		request.ContentType = newInfo.ContentType
 		request.Info = newInfo.Info
 	}
-	opts := rest.Opts{
-		Method:  "POST",
-		Path:    "/b2_start_large_file",
-		Options: optionsToSend,
-	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)

backend/box/box.go

@@ -43,9 +43,9 @@ import (
 	"github.com/rclone/rclone/lib/jwtutil"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -64,10 +64,12 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
-		Scopes:       nil,
-		AuthURL:      "https://app.box.com/api/oauth2/authorize",
-		TokenURL:     "https://app.box.com/api/oauth2/token",
+	oauthConfig = &oauth2.Config{
+		Scopes: nil,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://app.box.com/api/oauth2/authorize",
+			TokenURL: "https://app.box.com/api/oauth2/token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -254,10 +256,8 @@ func getQueryParams(boxConfig *api.ConfigJSON) map[string]string {
 }
 
 func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) {
+
 	block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey))
-	if block == nil {
-		return nil, errors.New("box: failed to PEM decode private key")
-	}
 	if len(rest) > 0 {
 		return nil, fmt.Errorf("box: extra data included in private key: %w", err)
 	}
@@ -619,7 +619,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		// fmt.Printf("...Error %v\n", err)
+		//fmt.Printf("...Error %v\n", err)
 		return "", err
 	}
 	// fmt.Printf("...Id %q\n", *info.Id)
@@ -966,26 +966,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	// check if dest already exists
-	item, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
-	if err != nil {
-		return nil, err
-	}
-	if item != nil { // dest already exists, need to copy to temp name and then move
-		tempSuffix := "-rclone-copy-" + random.String(8)
-		fs.Debugf(remote, "dst already exists, copying to temp name %v", remote+tempSuffix)
-		tempObj, err := f.Copy(ctx, src, remote+tempSuffix)
-		if err != nil {
-			return nil, err
-		}
-		fs.Debugf(remote+tempSuffix, "moving to real name %v", remote)
-		err = f.deleteObject(ctx, item.ID)
-		if err != nil {
-			return nil, err
-		}
-		return f.Move(ctx, tempObj, remote)
-	}
-
 	// Copy the object
 	opts := rest.Opts{
 		Method:     "POST",
@@ -1227,12 +1207,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 	return err
 }
 
-// Shutdown shutdown the fs
-func (f *Fs) Shutdown(ctx context.Context) error {
-	f.tokenRenewer.Shutdown()
-	return nil
-}
-
 // ChangeNotify calls the passed function with a path that has had changes.
 // If the implementation uses polling, it should adhere to the given interval.
 //
@@ -1745,7 +1719,6 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
-	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/cache/cache.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 // Package cache implements a virtual provider to cache existing remotes.
 package cache
@@ -409,16 +410,18 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
 			if err != nil {
 				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
 			}
-		} else if opt.PlexPassword != "" && opt.PlexUsername != "" {
-			decPass, err := obscure.Reveal(opt.PlexPassword)
-			if err != nil {
-				decPass = opt.PlexPassword
-			}
-			f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
-				m.Set("plex_token", token)
-			})
-			if err != nil {
-				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
+		} else {
+			if opt.PlexPassword != "" && opt.PlexUsername != "" {
+				decPass, err := obscure.Reveal(opt.PlexPassword)
+				if err != nil {
+					decPass = opt.PlexPassword
+				}
+				f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
+					m.Set("plex_token", token)
+				})
+				if err != nil {
+					return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
+				}
 			}
 		}
 	}

backend/cache/cache_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 
@@ -10,6 +11,7 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
+	"log"
 	"math/rand"
 	"os"
 	"path"
@@ -28,11 +30,10 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/vfs/vfscommon"
+	"github.com/rclone/rclone/vfs/vfsflags"
 	"github.com/stretchr/testify/require"
 )
 
@@ -92,7 +93,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
-	fs.Logf(nil, "Running with the following params: \n remote: %v", remoteName)
+	log.Printf("Running with the following params: \n remote: %v", remoteName)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -122,10 +123,10 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 
 /* TODO: is this testing something?
 func TestInternalVfsCache(t *testing.T) {
-	vfscommon.Opt.DirCacheTime = time.Second * 30
+	vfsflags.Opt.DirCacheTime = time.Second * 30
 	testSize := int64(524288000)
 
-	vfscommon.Opt.CacheMode = vfs.CacheModeWrites
+	vfsflags.Opt.CacheMode = vfs.CacheModeWrites
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -337,7 +338,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
+	vfsflags.Opt.DirCacheTime = time.Second
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -367,7 +368,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
+	vfsflags.Opt.DirCacheTime = time.Second
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -407,7 +408,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	// update in the wrapped fs
 	originalSize, err := runInstance.size(t, rootFs, "data.bin")
 	require.NoError(t, err)
-	fs.Logf(nil, "original size: %v", originalSize)
+	log.Printf("original size: %v", originalSize)
 
 	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
@@ -416,7 +417,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	if runInstance.rootIsCrypt {
 		data2, err = base64.StdEncoding.DecodeString(cryptedText3Base64)
 		require.NoError(t, err)
-		expectedSize++ // FIXME newline gets in, likely test data issue
+		expectedSize = expectedSize + 1 // FIXME newline gets in, likely test data issue
 	} else {
 		data2 = []byte("test content")
 	}
@@ -424,7 +425,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data2)), o.Size())
-	fs.Logf(nil, "updated size: %v", len(data2))
+	log.Printf("updated size: %v", len(data2))
 
 	// get a new instance from the cache
 	if runInstance.wrappedIsExternal {
@@ -484,49 +485,49 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		li, err := runInstance.list(t, rootFs, "test")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 2 {
-			fs.Logf(nil, "not expected listing /test: %v", li)
+			log.Printf("not expected listing /test: %v", li)
 			return fmt.Errorf("not expected listing /test: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 0 {
-			fs.Logf(nil, "not expected listing /test/one: %v", li)
+			log.Printf("not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/second")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			fs.Logf(nil, "not expected listing /test/second: %v", li)
+			log.Printf("not expected listing /test/second: %v", li)
 			return fmt.Errorf("not expected listing /test/second: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "data.bin" {
-				fs.Logf(nil, "not expected name: %v", fi.Name())
+				log.Printf("not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/second/data.bin" {
-				fs.Logf(nil, "not expected remote: %v", di.Remote())
+				log.Printf("not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			fs.Logf(nil, "unexpected listing: %v", li)
+			log.Printf("unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
 
-		fs.Logf(nil, "complete listing: %v", li)
+		log.Printf("complete listing: %v", li)
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -576,43 +577,43 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
 		if !found {
-			fs.Logf(nil, "not found /test")
+			log.Printf("not found /test")
 			return fmt.Errorf("not found /test")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one")))
 		if !found {
-			fs.Logf(nil, "not found /test/one")
+			log.Printf("not found /test/one")
 			return fmt.Errorf("not found /test/one")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2")))
 		if !found {
-			fs.Logf(nil, "not found /test/one/test2")
+			log.Printf("not found /test/one/test2")
 			return fmt.Errorf("not found /test/one/test2")
 		}
 		li, err := runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			fs.Logf(nil, "not expected listing /test/one: %v", li)
+			log.Printf("not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "test2" {
-				fs.Logf(nil, "not expected name: %v", fi.Name())
+				log.Printf("not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/one/test2" {
-				fs.Logf(nil, "not expected remote: %v", di.Remote())
+				log.Printf("not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			fs.Logf(nil, "unexpected listing: %v", li)
+			log.Printf("unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
-		fs.Logf(nil, "complete listing /test/one/test2")
+		log.Printf("complete listing /test/one/test2")
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -707,7 +708,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 4) // needs to be lower than the defined
+	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -742,7 +743,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 }
 
 func TestInternalBug2117(t *testing.T) {
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 10)
+	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
@@ -770,24 +771,24 @@ func TestInternalBug2117(t *testing.T) {
 
 	di, err := runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 1)
 
 	time.Sleep(time.Second * 30)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 1)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 4)
 
 	di, err = runInstance.list(t, rootFs, "test")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 4)
 }
 
@@ -828,7 +829,7 @@ func newRun() *run {
 	} else {
 		r.tmpUploadDir = uploadDir
 	}
-	fs.Logf(nil, "Temp Upload Dir: %v", r.tmpUploadDir)
+	log.Printf("Temp Upload Dir: %v", r.tmpUploadDir)
 
 	return r
 }
@@ -849,8 +850,8 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
-	for _, s := range config.GetRemotes() {
-		if s.Name == remote {
+	for _, s := range config.FileSections() {
+		if s == remote {
 			remoteExists = true
 		}
 	}
@@ -874,12 +875,12 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	cacheRemote := remote
 	if !remoteExists {
 		localRemote := remote + "-local"
-		config.FileSetValue(localRemote, "type", "local")
-		config.FileSetValue(localRemote, "nounc", "true")
+		config.FileSet(localRemote, "type", "local")
+		config.FileSet(localRemote, "nounc", "true")
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.GetValue(remote, "type")
+		remoteType := config.FileGet(remote, "type")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -890,14 +891,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.GetValue(remote, "remote")
+			remoteRemote := config.FileGet(remote, "remote")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.GetValue(remoteWrapping, "type")
+			remoteType := config.FileGet(remoteWrapping, "type")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -934,7 +935,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = operations.Purge(context.Background(), f, "")
+		_ = f.Features().Purge(context.Background(), "")
+		require.NoError(t, err)
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
@@ -947,7 +949,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 }
 
 func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
-	err := operations.Purge(context.Background(), f, "")
+	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
@@ -1191,7 +1193,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 func (r *run) cleanSize(t *testing.T, size int64) int64 {
 	if r.rootIsCrypt {
 		denominator := int64(65536 + 16)
-		size -= 32
+		size = size - 32
 		quotient := size / denominator
 		remainder := size % denominator
 		return (quotient*65536 + remainder - 16)

backend/cache/cache_test.go

@@ -1,6 +1,7 @@
 // Test Cache filesystem interface
 
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 
@@ -15,11 +16,10 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                      "TestCache:",
-		NilObject:                       (*cache.Object)(nil),
-		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"},
-		UnimplementableObjectMethods:    []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata", "SetMetadata"},
-		UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"},
-		SkipInvalidUTF8:                 true, // invalid UTF-8 confuses the cache
+		RemoteName:                   "TestCache:",
+		NilObject:                    (*cache.Object)(nil),
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
+		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || js
+// +build plan9 js
 
-// Package cache implements a virtual provider to cache existing remotes.
 package cache

backend/cache/cache_upload_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 

backend/cache/directory.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/handle.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 
@@ -118,7 +119,7 @@ func (r *Handle) startReadWorkers() {
 	r.scaleWorkers(totalWorkers)
 }
 
-// scaleWorkers will increase the worker pool count by the provided amount
+// scaleOutWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
 	current := r.workers
 	if current == desired {
@@ -208,7 +209,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	offset := chunkStart % int64(r.cacheFs().opt.ChunkSize)
 
 	// we align the start offset of the first chunk to a likely chunk in the storage
-	chunkStart -= offset
+	chunkStart = chunkStart - offset
 	r.queueOffset(chunkStart)
 	found := false
 
@@ -327,7 +328,7 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 
 	chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize))
 	if chunkStart >= int64(r.cacheFs().opt.ChunkSize) {
-		chunkStart -= int64(r.cacheFs().opt.ChunkSize)
+		chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize)
 	}
 	r.queueOffset(chunkStart)
 
@@ -415,8 +416,10 @@ func (w *worker) run() {
 					continue
 				}
 			}
-		} else if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
-			continue
+		} else {
+			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
+				continue
+			}
 		}
 
 		chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize)

backend/cache/object.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/plex.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/storage_memory.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/storage_persistent.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/utils_test.go

@@ -1,6 +1,3 @@
-//go:build !plan9 && !js
-// +build !plan9,!js
-
 package cache
 
 import bolt "go.etcd.io/bbolt"

backend/chunker/chunker.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/lib/encoder"
 )
 
 // Chunker's composite files have one or more chunks
@@ -102,10 +101,8 @@ var (
 //
 // And still chunker's primary function is to chunk large files
 // rather than serve as a generic metadata container.
-const (
-	maxMetadataSize        = 1023
-	maxMetadataSizeWritten = 255
-)
+const maxMetadataSize = 1023
+const maxMetadataSizeWritten = 255
 
 // Current/highest supported metadata format.
 const metadataVersion = 2
@@ -308,6 +305,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		root: rpath,
 		opt:  *opt,
 	}
+	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType, opt.Transactions); err != nil {
@@ -319,15 +317,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// i.e. `rpath` does not exist in the wrapped remote, but chunker
 	// detects a composite file because it finds the first chunk!
 	// (yet can't satisfy fstest.CheckListing, will ignore)
-	if err == nil && !f.useMeta {
+	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		newBase, testErr := cache.Get(ctx, baseName+firstChunkPath)
+		_, testErr := cache.Get(ctx, baseName+firstChunkPath)
 		if testErr == fs.ErrorIsFile {
-			f.base = newBase
 			err = testErr
 		}
 	}
-	cache.PinUntilFinalized(f.base, f)
 
 	// Correct root if definitely pointing to a file
 	if err == fs.ErrorIsFile {
@@ -342,18 +338,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// Note 2: features.Fill() points features.PutStream to our PutStream,
 	// but features.Mask() will nullify it if wrappedFs does not have it.
 	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           true,
-		ReadMimeType:             false, // Object.MimeType not supported
-		WriteMimeType:            true,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		ServerSideAcrossConfigs:  true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          true,
+		ReadMimeType:            false, // Object.MimeType not supported
+		WriteMimeType:           true,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: true,
 	}).Fill(ctx, f).Mask(ctx, baseFs).WrapsFs(f, baseFs)
 
 	f.features.Disable("ListR") // Recursive listing may cause chunker skip files
@@ -830,7 +821,8 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 			}
 		case fs.Directory:
 			isSubdir[entry.Remote()] = true
-			wrapDir := fs.NewDirWrapper(entry.Remote(), entry)
+			wrapDir := fs.NewDirCopy(ctx, entry)
+			wrapDir.SetRemote(entry.Remote())
 			tempEntries = append(tempEntries, wrapDir)
 		default:
 			if f.opt.FailHard {
@@ -963,11 +955,6 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		}
 		if caseInsensitive {
 			sameMain = strings.EqualFold(mainRemote, remote)
-			if sameMain && f.base.Features().IsLocal {
-				// on local, make sure the EqualFold still holds true when accounting for encoding.
-				// sometimes paths with special characters will only normalize the same way in Standard Encoding.
-				sameMain = strings.EqualFold(encoder.OS.FromStandardPath(mainRemote), encoder.OS.FromStandardPath(remote))
-			}
 		} else {
 			sameMain = mainRemote == remote
 		}
@@ -981,13 +968,13 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 			}
 			continue
 		}
-		// fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
+		//fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
 		if err := o.addChunk(entry, chunkNo); err != nil {
 			return nil, err
 		}
 	}
 
-	if o.main == nil && len(o.chunks) == 0 {
+	if o.main == nil && (o.chunks == nil || len(o.chunks) == 0) {
 		// Scanning hasn't found data chunks with conforming names.
 		if f.useMeta || quickScan {
 			// Metadata is required but absent and there are no chunks.
@@ -1143,8 +1130,8 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 // put implements Put, PutStream, PutUnchecked, Update
 func (f *Fs) put(
 	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
-	basePut putFn, action string, target fs.Object,
-) (obj fs.Object, err error) {
+	basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
+
 	// Perform consistency checks
 	if err := f.forbidChunk(src, remote); err != nil {
 		return nil, fmt.Errorf("%s refused: %w", action, err)
@@ -1584,14 +1571,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.base.Mkdir(ctx, dir)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.base.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -1909,14 +1888,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.base, srcRemote, dstRemote)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.base.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1965,7 +1936,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		return
 	}
 	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		//fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
 		if entryType == fs.EntryObject {
 			mainPath, _, _, xactID := f.parseChunkName(path)
 			metaXactID := ""
@@ -2480,7 +2451,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
 	if len(data) > maxMetadataSizeWritten {
 		return nil, false, ErrMetaTooBig
 	}
-	if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
+	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
 		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON
@@ -2577,8 +2548,6 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -36,7 +36,6 @@ func TestIntegration(t *testing.T) {
 			"GetTier",
 			"SetTier",
 			"Metadata",
-			"SetMetadata",
 		},
 		UnimplementableFsMethods: []string{
 			"PublicLink",

backend/cloudinary/api/types.go

@@ -1,48 +0,0 @@
-// Package api has type definitions for cloudinary
-package api
-
-import (
-	"fmt"
-)
-
-// CloudinaryEncoder extends the built-in encoder
-type CloudinaryEncoder interface {
-	// FromStandardPath takes a / separated path in Standard encoding
-	// and converts it to a / separated path in this encoding.
-	FromStandardPath(string) string
-	// FromStandardName takes name in Standard encoding and converts
-	// it in this encoding.
-	FromStandardName(string) string
-	// ToStandardPath takes a / separated path in this encoding
-	// and converts it to a / separated path in Standard encoding.
-	ToStandardPath(string) string
-	// ToStandardName takes name in this encoding and converts
-	// it in Standard encoding.
-	ToStandardName(string) string
-	// Encoded root of the remote (as passed into NewFs)
-	FromStandardFullPath(string) string
-}
-
-// UpdateOptions was created to pass options from Update to Put
-type UpdateOptions struct {
-	PublicID     string
-	ResourceType string
-	DeliveryType string
-	AssetFolder  string
-	DisplayName  string
-}
-
-// Header formats the option as a string
-func (o *UpdateOptions) Header() (string, string) {
-	return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}
-
-// Mandatory returns whether the option must be parsed or can be ignored
-func (o *UpdateOptions) Mandatory() bool {
-	return false
-}
-
-// String formats the option into human-readable form
-func (o *UpdateOptions) String() string {
-	return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
-}

backend/cloudinary/cloudinary.go

@@ -1,711 +0,0 @@
-// Package cloudinary provides an interface to the Cloudinary DAM
-package cloudinary
-
-import (
-	"context"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cloudinary/cloudinary-go/v2"
-	SDKApi "github.com/cloudinary/cloudinary-go/v2/api"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin"
-	"github.com/cloudinary/cloudinary-go/v2/api/admin/search"
-	"github.com/cloudinary/cloudinary-go/v2/api/uploader"
-	"github.com/rclone/rclone/backend/cloudinary/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-	"github.com/zeebo/blake3"
-)
-
-// Cloudinary shouldn't have a trailing dot if there is no path
-func cldPathDir(somePath string) string {
-	if somePath == "" || somePath == "." {
-		return somePath
-	}
-	dir := path.Dir(somePath)
-	if dir == "." {
-		return ""
-	}
-	return dir
-}
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "cloudinary",
-		Description: "Cloudinary",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:      "cloud_name",
-				Help:      "Cloudinary Environment Name",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_key",
-				Help:      "Cloudinary API Key",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name:      "api_secret",
-				Help:      "Cloudinary API Secret",
-				Required:  true,
-				Sensitive: true,
-			},
-			{
-				Name: "upload_prefix",
-				Help: "Specify the API endpoint for environments out of the US",
-			},
-			{
-				Name: "upload_preset",
-				Help: "Upload Preset to select asset manipulation on upload",
-			},
-			{
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
-					encoder.EncodeSlash |
-					encoder.EncodeLtGt |
-					encoder.EncodeDoubleQuote |
-					encoder.EncodeQuestion |
-					encoder.EncodeAsterisk |
-					encoder.EncodePipe |
-					encoder.EncodeHash |
-					encoder.EncodePercent |
-					encoder.EncodeBackSlash |
-					encoder.EncodeDel |
-					encoder.EncodeCtl |
-					encoder.EncodeRightSpace |
-					encoder.EncodeInvalidUtf8 |
-					encoder.EncodeDot),
-			},
-			{
-				Name:     "eventually_consistent_delay",
-				Default:  fs.Duration(0),
-				Advanced: true,
-				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",
-			},
-		},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	CloudName                 string               `config:"cloud_name"`
-	APIKey                    string               `config:"api_key"`
-	APISecret                 string               `config:"api_secret"`
-	UploadPrefix              string               `config:"upload_prefix"`
-	UploadPreset              string               `config:"upload_preset"`
-	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EventuallyConsistentDelay fs.Duration          `config:"eventually_consistent_delay"`
-}
-
-// Fs represents a remote cloudinary server
-type Fs struct {
-	name     string
-	root     string
-	opt      Options
-	features *fs.Features
-	pacer    *fs.Pacer
-	srv      *rest.Client           // For downloading assets via the Cloudinary CDN
-	cld      *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK
-	lastCRUD time.Time
-}
-
-// Object describes a cloudinary object
-type Object struct {
-	fs           *Fs
-	remote       string
-	size         int64
-	modTime      time.Time
-	url          string
-	md5sum       string
-	publicID     string
-	resourceType string
-	deliveryType string
-}
-
-// NewFs constructs an Fs from the path, bucket:path
-func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize the Cloudinary client
-	cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)
-	}
-	cld.Admin.Client = *fshttp.NewClient(ctx)
-	cld.Upload.Client = *fshttp.NewClient(ctx)
-	if opt.UploadPrefix != "" {
-		cld.Config.API.UploadPrefix = opt.UploadPrefix
-	}
-	client := fshttp.NewClient(ctx)
-	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		cld:   cld,
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
-		srv:   rest.NewClient(client),
-	}
-
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-	}).Fill(ctx, f)
-
-	if root != "" {
-		// Check to see if the root actually an existing file
-		remote := path.Base(root)
-		f.root = cldPathDir(root)
-		_, err := f.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
-				// File doesn't exist so return the previous root
-				f.root = root
-				return f, nil
-			}
-			return nil, err
-		}
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-// ------------------------------------------------------------
-
-// FromStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")
-}
-
-// FromStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) FromStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
-}
-
-// ToStandardPath implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardPath(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")
-}
-
-// ToStandardName implementation of the api.CloudinaryEncoder
-func (f *Fs) ToStandardName(s string) string {
-	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
-}
-
-// FromStandardFullPath encodes a full path to Cloudinary standard
-func (f *Fs) FromStandardFullPath(dir string) string {
-	return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))
-}
-
-// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK
-func (f *Fs) ToAssetFolderAPI(dir string) string {
-	return strings.ReplaceAll(dir, "%", "%25")
-}
-
-// ToDisplayNameElastic encodes a special case of elasticsearch
-func (f *Fs) ToDisplayNameElastic(dir string) string {
-	return strings.ReplaceAll(dir, "!", "\\!")
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// WaitEventuallyConsistent waits till the FS is eventually consistent
-func (f *Fs) WaitEventuallyConsistent() {
-	if f.opt.EventuallyConsistentDelay == fs.Duration(0) {
-		return
-	}
-	delay := time.Duration(f.opt.EventuallyConsistentDelay)
-	timeSinceLastCRUD := time.Since(f.lastCRUD)
-	if timeSinceLastCRUD < delay {
-		time.Sleep(delay - timeSinceLastCRUD)
-	}
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("Cloudinary root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// List the objects and directories in dir into entries
-func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
-	remotePrefix := f.FromStandardFullPath(dir)
-	if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {
-		remotePrefix += "/"
-	}
-
-	var entries fs.DirEntries
-	dirs := make(map[string]struct{})
-	nextCursor := ""
-	f.WaitEventuallyConsistent()
-	for {
-		// user the folders api to list folders.
-		folderParams := admin.SubFoldersParams{
-			Folder:     f.ToAssetFolderAPI(remotePrefix),
-			MaxResults: 500,
-		}
-		if nextCursor != "" {
-			folderParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list sub-folders: %w", err)
-		}
-		if results.Error.Message != "" {
-			if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {
-				return nil, fs.ErrorDirNotFound
-			}
-
-			return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)
-		}
-
-		for _, folder := range results.Folders {
-			relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))
-			parts := strings.Split(relativePath, "/")
-
-			// It's a directory
-			dirName := parts[len(parts)-1]
-			if _, found := dirs[dirName]; !found {
-				d := fs.NewDir(path.Join(dir, dirName), time.Time{})
-				entries = append(entries, d)
-				dirs[dirName] = struct{}{}
-			}
-		}
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	for {
-		// Use the assets.AssetsByAssetFolder API to list assets
-		assetsParams := admin.AssetsByAssetFolderParams{
-			AssetFolder: remotePrefix,
-			MaxResults:  500,
-		}
-		if nextCursor != "" {
-			assetsParams.NextCursor = nextCursor
-		}
-
-		results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list assets: %w", err)
-		}
-
-		for _, asset := range results.Assets {
-			remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
-			if dir != "" {
-				remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
-			}
-			o := &Object{
-				fs:           f,
-				remote:       remote,
-				size:         int64(asset.Bytes),
-				modTime:      asset.CreatedAt,
-				url:          asset.SecureURL,
-				publicID:     asset.PublicID,
-				resourceType: asset.AssetType,
-				deliveryType: asset.Type,
-			}
-			entries = append(entries, o)
-		}
-
-		// Break if there are no more results
-		if results.NextCursor == "" {
-			break
-		}
-		nextCursor = results.NextCursor
-	}
-
-	return entries, nil
-}
-
-// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	searchParams := search.Query{
-		Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",
-			f.FromStandardFullPath(cldPathDir(remote)),
-			f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),
-		SortBy:     []search.SortByField{{"uploaded_at": "desc"}},
-		MaxResults: 2,
-	}
-	var results *admin.SearchResult
-	f.WaitEventuallyConsistent()
-	err := f.pacer.Call(func() (bool, error) {
-		var err1 error
-		results, err1 = f.cld.Admin.Search(ctx, searchParams)
-		if err1 == nil && results.TotalCount != len(results.Assets) {
-			err1 = errors.New("partial response so waiting for eventual consistency")
-		}
-		return shouldRetry(ctx, nil, err1)
-	})
-	if err != nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-	if results.TotalCount == 0 || len(results.Assets) == 0 {
-		return nil, fs.ErrorObjectNotFound
-	}
-	asset := results.Assets[0]
-
-	o := &Object{
-		fs:           f,
-		remote:       remote,
-		size:         int64(asset.Bytes),
-		modTime:      asset.UploadedAt,
-		url:          asset.SecureURL,
-		md5sum:       asset.Etag,
-		publicID:     asset.PublicID,
-		resourceType: asset.ResourceType,
-		deliveryType: asset.Type,
-	}
-
-	return o, nil
-}
-
-func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {
-	payload := []byte(path.Join(assetFolder, displayName))
-	hash := blake3.Sum256(payload)
-	return hex.EncodeToString(hash[:])
-}
-
-// Put uploads content to Cloudinary
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if src.Size() == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	params := uploader.UploadParams{
-		UploadPreset: f.opt.UploadPreset,
-	}
-
-	updateObject := false
-	var modTime time.Time
-	for _, option := range options {
-		if updateOptions, ok := option.(*api.UpdateOptions); ok {
-			if updateOptions.PublicID != "" {
-				updateObject = true
-				params.Overwrite = SDKApi.Bool(true)
-				params.Invalidate = SDKApi.Bool(true)
-				params.PublicID = updateOptions.PublicID
-				params.ResourceType = updateOptions.ResourceType
-				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
-				params.AssetFolder = updateOptions.AssetFolder
-				params.DisplayName = updateOptions.DisplayName
-				modTime = src.ModTime(ctx)
-			}
-		}
-	}
-	if !updateObject {
-		params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))
-		params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))
-		// We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).
-		// We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.
-		// Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.
-		params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))
-	}
-	uploadResult, err := f.cld.Upload.Upload(ctx, in, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)
-	}
-	if !updateObject {
-		modTime = uploadResult.CreatedAt
-	}
-	if uploadResult.Error.Message != "" {
-		return nil, errors.New(uploadResult.Error.Message)
-	}
-
-	o := &Object{
-		fs:           f,
-		remote:       src.Remote(),
-		size:         int64(uploadResult.Bytes),
-		modTime:      modTime,
-		url:          uploadResult.SecureURL,
-		md5sum:       uploadResult.Etag,
-		publicID:     uploadResult.PublicID,
-		resourceType: uploadResult.ResourceType,
-		deliveryType: uploadResult.Type,
-	}
-	return o, nil
-}
-
-// Precision of the remote
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash sets
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// Mkdir creates empty folders
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}
-	res, err := f.cld.Admin.CreateFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// Rmdir deletes empty folders
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	// Additional test because Cloudinary will delete folders without
-	// assets, regardless of empty sub-folders
-	folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))
-	folderParams := admin.SubFoldersParams{
-		Folder:     folder,
-		MaxResults: 1,
-	}
-	results, err := f.cld.Admin.SubFolders(ctx, folderParams)
-	if err != nil {
-		return err
-	}
-	if results.TotalCount > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-
-	params := admin.DeleteFolderParams{Folder: folder}
-	res, err := f.cld.Admin.DeleteFolder(ctx, params)
-	f.lastCRUD = time.Now()
-	if err != nil {
-		return err
-	}
-	if res.Error.Message != "" {
-		if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {
-			return fs.ErrorDirNotFound
-		}
-
-		return errors.New(res.Error.Message)
-	}
-
-	return nil
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	420, // Too Many Requests (legacy)
-	429, // Too Many Requests
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	if err != nil {
-		tryAgain := "Try again on "
-		if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
-			layout := "2006-01-02 15:04:05 UTC"
-			dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
-			timestamp, err2 := time.Parse(layout, dateStr)
-			if err2 == nil {
-				return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
-			}
-		}
-
-		fs.Debugf(nil, "Retrying API error %v", err)
-		return true, err
-	}
-
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-// ------------------------------------------------------------
-
-// Hash returns the MD5 of an object
-func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
-	if ty != hash.MD5 {
-		return "", hash.ErrUnsupported
-	}
-	return o.md5sum, nil
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the object
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// Size of object in bytes
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// Storable returns if this object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-// Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: o.url,
-		Options: options,
-	}
-	var offset int64
-	var count int64
-	var key string
-	var value string
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-			key, value = option.Header()
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size - offset
-			key, value = option.Header()
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if key != "" && value != "" {
-		opts.ExtraHeaders = make(map[string]string)
-		opts.ExtraHeaders[key] = value
-	}
-	// Make sure that the asset is fully available
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
-		if err == nil {
-			cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))
-			if clErr == nil && count == int64(cl) {
-				return false, nil
-			}
-		}
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)
-	}
-	return resp.Body, err
-}
-
-// Update the object with the contents of the io.Reader
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	options = append(options, &api.UpdateOptions{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		DeliveryType: o.deliveryType,
-		DisplayName:  api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),
-		AssetFolder:  o.fs.FromStandardFullPath(cldPathDir(o.Remote())),
-	})
-	updatedObj, err := o.fs.Put(ctx, in, src, options...)
-	if err != nil {
-		return err
-	}
-	if uo, ok := updatedObj.(*Object); ok {
-		o.size = uo.size
-		o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time
-		o.url = uo.url
-		o.md5sum = uo.md5sum
-		o.publicID = uo.publicID
-		o.resourceType = uo.resourceType
-		o.deliveryType = uo.deliveryType
-	}
-	return nil
-}
-
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	params := uploader.DestroyParams{
-		PublicID:     o.publicID,
-		ResourceType: o.resourceType,
-		Type:         o.deliveryType,
-	}
-	res, dErr := o.fs.cld.Upload.Destroy(ctx, params)
-	o.fs.lastCRUD = time.Now()
-	if dErr != nil {
-		return dErr
-	}
-
-	if res.Error.Message != "" {
-		return errors.New(res.Error.Message)
-	}
-
-	if res.Result != "ok" {
-		return errors.New(res.Result)
-	}
-
-	return nil
-}

backend/cloudinary/cloudinary_test.go

@@ -1,23 +0,0 @@
-// Test Cloudinary filesystem interface
-
-package cloudinary_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/cloudinary"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	name := "TestCloudinary"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      name + ":",
-		NilObject:       (*cloudinary.Object)(nil),
-		SkipInvalidUTF8: true,
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "eventually_consistent_delay", Value: "7"},
-		},
-	})
-}

backend/combine/combine.go

@@ -222,23 +222,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 	}
 	// check features
 	var features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           false,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		CanHaveEmptyDirectories:  true,
-		BucketBased:              true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		BucketBased:             true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f)
 	canMove := true
 	for _, u := range f.upstreams {
@@ -445,32 +440,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return u.f.Mkdir(ctx, uRemote)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	u, uRemote, err := f.findUpstream(dir)
-	if err != nil {
-		return nil, err
-	}
-	do := u.f.Features().MkdirMetadata
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	newDir, err := do(ctx, uRemote, metadata)
-	if err != nil {
-		return nil, err
-	}
-	entries := fs.DirEntries{newDir}
-	entries, err = u.wrapEntries(ctx, entries)
-	if err != nil {
-		return nil, err
-	}
-	newDir, ok := entries[0].(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
-	}
-	return newDir, nil
-}
-
 // purge the upstream or fallback to a slow way
 func (u *upstream) purge(ctx context.Context, dir string) (err error) {
 	if do := u.f.Features().Purge; do != nil {
@@ -786,11 +755,12 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D
 		case fs.Object:
 			entries[i] = u.newObject(x)
 		case fs.Directory:
-			newPath, err := u.pathAdjustment.do(x.Remote())
+			newDir := fs.NewDirCopy(ctx, x)
+			newPath, err := u.pathAdjustment.do(newDir.Remote())
 			if err != nil {
 				return nil, err
 			}
-			newDir := fs.NewDirWrapper(newPath, x)
+			newDir.SetRemote(newPath)
 			entries[i] = newDir
 		default:
 			return nil, fmt.Errorf("unknown entry type %T", entry)
@@ -813,7 +783,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if f.root == "" && dir == "" {
 		entries = make(fs.DirEntries, 0, len(f.upstreams))
 		for combineDir := range f.upstreams {
-			d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
+			d := fs.NewDir(combineDir, f.when)
 			entries = append(entries, d)
 		}
 		return entries, nil
@@ -995,22 +965,6 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return do(ctx, uDirs)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	u, uDir, err := f.findUpstream(dir)
-	if err != nil {
-		return err
-	}
-	if uDir == "" {
-		fs.Debugf(dir, "Can't set modtime on upstream root. skipping.")
-		return nil
-	}
-	if do := u.f.Features().DirSetModTime; do != nil {
-		return do(ctx, uDir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1119,17 +1073,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // SetTier performs changing storage tier of the Object if
 // multiple storage classes supported
 func (o *Object) SetTier(tier string) error {
@@ -1156,8 +1099,6 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.OpenWriterAter  = (*Fs)(nil)
 	_ fs.FullObject      = (*Object)(nil)

backend/compress/compress.go

@@ -38,7 +38,6 @@ import (
 const (
 	initialChunkSize = 262144  // Initial and max sizes of chunks when reading parts of the file. Currently
 	maxChunkSize     = 8388608 // at 256 KiB and 8 MiB.
-	chunkStreams     = 0       // Streams to use for reading
 
 	bufferSize          = 8388608
 	heuristicBytes      = 1048576
@@ -184,23 +183,18 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           false,
-		ReadMimeType:             false,
-		WriteMimeType:            false,
-		GetTier:                  true,
-		SetTier:                  true,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            false,
+		WriteMimeType:           false,
+		GetTier:                 true,
+		SetTier:                 true,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -456,7 +450,7 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Mul
 		if err != nil {
 			fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 		}
-		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
+		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ %q vs %q", ht, srcHash, dstHash)
 	}
 	return nil
 }
@@ -790,14 +784,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, dir)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.Fs.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -941,14 +927,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.Fs, srcRemote, dstRemote)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.Fs.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1287,17 +1265,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
@@ -1363,7 +1330,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 		}
 	}
 	// Get a chunkedreader for the wrapped object
-	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)
+	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize)
 	// Get file handle
 	var file io.Reader
 	if offset != 0 {
@@ -1530,8 +1497,6 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.UnWrapper       = (*Fs)(nil)

backend/crypt/cipher.go

@@ -329,7 +329,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	for _, runeValue := range plaintext {
 		dir += int(runeValue)
 	}
-	dir %= 256
+	dir = dir % 256
 
 	// We'll use this number to store in the result filename...
 	var result bytes.Buffer
@@ -450,7 +450,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if pos >= 26 {
 				pos -= 6
 			}
-			pos -= thisdir
+			pos = pos - thisdir
 			if pos < 0 {
 				pos += 52
 			}
@@ -888,7 +888,7 @@ func (fh *decrypter) fillBuffer() (err error) {
 		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
 		// Zero out the bad block and continue
 		for i := range (*fh.buf)[:n] {
-			fh.buf[i] = 0
+			(*fh.buf)[i] = 0
 		}
 	}
 	fh.bufIndex = 0

backend/crypt/crypt.go

@@ -130,16 +130,6 @@ trying to recover an encrypted file with errors and it is desired to
 recover as much of the file as possible.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "strict_names",
-			Help: `If set, this will raise an error when crypt comes across a filename that can't be decrypted.
-
-(By default, rclone will just log a NOTICE and continue as normal.)
-This can happen if encrypted and unencrypted files are stored in the same
-directory (which is not recommended.) It may also indicate a more serious
-problem that should be investigated.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -273,24 +263,19 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:          !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
-		DuplicateFiles:           true,
-		ReadMimeType:             false, // MimeTypes not supported with crypt
-		WriteMimeType:            false,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
+		DuplicateFiles:          true,
+		ReadMimeType:            false, // MimeTypes not supported with crypt
+		WriteMimeType:           false,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -309,7 +294,6 @@ type Options struct {
 	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
 	Suffix                  string `config:"suffix"`
-	StrictNames             bool   `config:"strict_names"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -344,64 +328,45 @@ func (f *Fs) String() string {
 }
 
 // Encrypt an object file name to entries.
-func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) error {
+func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	remote := obj.Remote()
 	decryptedRemote, err := f.cipher.DecryptFileName(remote)
 	if err != nil {
-		if f.opt.StrictNames {
-			return fmt.Errorf("%s: undecryptable file name detected: %v", remote, err)
-		}
-		fs.Logf(remote, "Skipping undecryptable file name: %v", err)
-		return nil
+		fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
+		return
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newObject(obj))
-	return nil
 }
 
 // Encrypt a directory file name to entries.
-func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) error {
+func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
-		if f.opt.StrictNames {
-			return fmt.Errorf("%s: undecryptable dir name detected: %v", remote, err)
-		}
-		fs.Logf(remote, "Skipping undecryptable dir name: %v", err)
-		return nil
+		fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
+		return
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newDir(ctx, dir))
-	return nil
 }
 
 // Encrypt some directory entries.  This alters entries returning it as newEntries.
 func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
-	errors := 0
-	var firsterr error
 	for _, entry := range entries {
 		switch x := entry.(type) {
 		case fs.Object:
-			err = f.add(&newEntries, x)
+			f.add(&newEntries, x)
 		case fs.Directory:
-			err = f.addDir(ctx, &newEntries, x)
+			f.addDir(ctx, &newEntries, x)
 		default:
 			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
-		if err != nil {
-			errors++
-			if firsterr == nil {
-				firsterr = err
-			}
-		}
-	}
-	if firsterr != nil {
-		return nil, fmt.Errorf("there were %v undecryptable name errors. first error: %v", errors, firsterr)
 	}
 	return newEntries, nil
 }
@@ -520,7 +485,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -555,37 +520,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, f.cipher.EncryptDirName(dir))
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	do := f.Fs.Features().MkdirMetadata
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	newDir, err := do(ctx, f.cipher.EncryptDirName(dir), metadata)
-	if err != nil {
-		return nil, err
-	}
-	var entries = make(fs.DirEntries, 0, 1)
-	err = f.addDir(ctx, &entries, newDir)
-	if err != nil {
-		return nil, err
-	}
-	newDir, ok := entries[0].(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
-	}
-	return newDir, nil
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	do := f.Fs.Features().DirSetModTime
-	if do == nil {
-		return fs.ErrorNotImplemented
-	}
-	return do(ctx, f.cipher.EncryptDirName(dir), modTime)
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -827,7 +761,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	}
 	out := make([]fs.Directory, len(dirs))
 	for i, dir := range dirs {
-		out[i] = fs.NewDirWrapper(f.cipher.EncryptDirName(dir.Remote()), dir)
+		out[i] = fs.NewDirCopy(ctx, dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
 	}
 	return do(ctx, out)
 }
@@ -1063,14 +997,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // newDir returns a dir with the Name decrypted
 func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory {
+	newDir := fs.NewDirCopy(ctx, dir)
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debugf(remote, "Undecryptable dir name: %v", err)
 	} else {
-		remote = decryptedRemote
+		newDir.SetRemote(decryptedRemote)
 	}
-	newDir := fs.NewDirWrapper(remote, dir)
 	return newDir
 }
 
@@ -1248,17 +1182,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // MimeType returns the content type of the Object if
 // known, or "" if not
 //
@@ -1284,8 +1207,6 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/drive/drive.go

@@ -80,10 +80,9 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	driveConfig = &oauthutil.Config{
+	driveConfig = &oauth2.Config{
 		Scopes:       []string{scopePrefix + "drive"},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
+		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -121,7 +120,6 @@ var (
 		"text/html":                 ".html",
 		"text/plain":                ".txt",
 		"text/tab-separated-values": ".tsv",
-		"text/markdown":             ".md",
 	}
 	_mimeTypeToExtensionLinks = map[string]string{
 		"application/x-link-desktop": ".desktop",
@@ -153,7 +151,6 @@ func (rwChoices) Choices() []fs.BitsChoicesInfo {
 		{Bit: uint64(rwOff), Name: "off"},
 		{Bit: uint64(rwRead), Name: "read"},
 		{Bit: uint64(rwWrite), Name: "write"},
-		{Bit: uint64(rwFailOK), Name: "failok"},
 	}
 }
 
@@ -163,7 +160,6 @@ type rwChoice = fs.Bits[rwChoices]
 const (
 	rwRead rwChoice = 1 << iota
 	rwWrite
-	rwFailOK
 	rwOff rwChoice = 0
 )
 
@@ -177,9 +173,6 @@ var rwExamples = fs.OptionExamples{{
 }, {
 	Value: rwWrite.String(),
 	Help:  "Write the value only",
-}, {
-	Value: rwFailOK.String(),
-	Help:  "If writing fails log errors only, don't fail the transfer",
 }, {
 	Value: (rwRead | rwWrite).String(),
 	Help:  "Read and Write the value.",
@@ -203,6 +196,7 @@ func driveScopesContainsAppFolder(scopes []string) bool {
 		if scope == scopePrefix+"drive.appfolder" {
 			return true
 		}
+
 	}
 	return false
 }
@@ -293,10 +287,7 @@ func init() {
 		},
 		MetadataInfo: &fs.MetadataInfo{
 			System: systemMetadataInfo,
-			Help: `User metadata is stored in the properties field of the drive object.
-
-Metadata is supported on files and directories.
-`,
+			Help:   `User metadata is stored in the properties field of the drive object.`,
 		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
@@ -879,11 +870,6 @@ type Object struct {
 	v2Download bool   // generate v2 download link ondemand
 }
 
-// Directory describes a drive directory
-type Directory struct {
-	baseObject
-}
-
 // ------------------------------------------------------------
 
 // Name of the remote (as passed into NewFs)
@@ -1211,7 +1197,6 @@ func fixMimeType(mimeTypeIn string) string {
 	}
 	return mimeTypeOut
 }
-
 func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	out = make(map[string][]string, len(in))
 	for k, v := range in {
@@ -1222,11 +1207,9 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
 	}
 	return out
 }
-
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
 }
-
 func isLinkMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/x-link-")
 }
@@ -1391,20 +1374,15 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
-		DuplicateFiles:           true,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		CanHaveEmptyDirectories:  true,
-		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
-		FilterAware:              true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: false, // FIXME need to check!
+		DuplicateFiles:          true,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
@@ -1659,8 +1637,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F
 // When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	ctx context.Context, remote string, info *drive.File,
-	extension, exportName, exportMimeType string, isDocument bool,
-) (o fs.Object, err error) {
+	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
 	// Note that resolveShortcut will have been called already if
 	// we are being called from a listing. However the drive.Item
 	// will have been resolved so this will do nothing.
@@ -1752,72 +1729,26 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	return pathIDOut, found, err
 }
 
-// createDir makes a directory with pathID as parent and name leaf with optional metadata
-func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Metadata) (info *drive.File, err error) {
-	leaf = f.opt.Enc.FromStandardName(leaf)
-	pathID = actualID(pathID)
-	createInfo := &drive.File{
-		Name:     leaf,
-		MimeType: driveFolderType,
-		Parents:  []string{pathID},
-	}
-	var updateMetadata updateMetadataFn
-	if len(metadata) > 0 {
-		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true)
-		if err != nil {
-			return nil, fmt.Errorf("create dir: failed to update metadata: %w", err)
-		}
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Create(createInfo).
-			Fields(f.getFileFields(ctx)).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	if updateMetadata != nil {
-		err = updateMetadata(ctx, info)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return info, nil
-}
-
-// updateDir updates an existing a directory with the metadata passed in
-func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata) (info *drive.File, err error) {
-	if len(metadata) == 0 {
-		return f.getFile(ctx, dirID, f.getFileFields(ctx))
-	}
-	dirID = actualID(dirID)
-	updateInfo := &drive.File{}
-	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true)
-	if err != nil {
-		return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err)
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Update(dirID, updateInfo).
-			Fields(f.getFileFields(ctx)).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
-	return info, nil
-}
-
 // CreateDir makes a directory with pathID as parent and name leaf
 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
-	info, err := f.createDir(ctx, pathID, leaf, nil)
+	leaf = f.opt.Enc.FromStandardName(leaf)
+	// fmt.Println("Making", path)
+	// Define the metadata for the directory we are going to create.
+	pathID = actualID(pathID)
+	createInfo := &drive.File{
+		Name:        leaf,
+		Description: leaf,
+		MimeType:    driveFolderType,
+		Parents:     []string{pathID},
+	}
+	var info *drive.File
+	err = f.pacer.Call(func() (bool, error) {
+		info, err = f.svc.Files.Create(createInfo).
+			Fields("id").
+			SupportsAllDrives(true).
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return "", err
 	}
@@ -1851,7 +1782,6 @@ func linkTemplate(mt string) *template.Template {
 	})
 	return _linkTemplates[mt]
 }
-
 func (f *Fs) fetchFormats(ctx context.Context) {
 	fetchFormatsOnce.Do(func() {
 		var about *drive.About
@@ -1897,8 +1827,7 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
 // Look through the exportExtensions and find the first format that can be
 // converted.  If none found then return ("", "", false)
 func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
-	extension, mimeType string, isDocument bool,
-) {
+	extension, mimeType string, isDocument bool) {
 	exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
 	if isDocument {
 		for _, _extension := range f.exportExtensions {
@@ -1930,7 +1859,7 @@ func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string
 	return "", "", isDocument
 }
 
-// findExportFormat works out the optimum export settings
+// findExportFormatByMimeType works out the optimum export settings
 // for the given drive.File.
 //
 // Look through the exportExtensions and find the first format that can be
@@ -2226,13 +2155,13 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		case in <- job:
 		default:
 			overflow = append(overflow, job)
-			wg.Done()
+			wg.Add(-1)
 		}
 	}
 
 	// Send the entry to the caller, queueing any directories as new jobs
 	cb := func(entry fs.DirEntry) error {
-		if d, isDir := entry.(fs.Directory); isDir {
+		if d, isDir := entry.(*fs.Dir); isDir {
 			job := listREntry{actualID(d.ID()), d.Remote()}
 			sendJob(job)
 		}
@@ -2409,11 +2338,11 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File
 		if item.ResourceKey != "" {
 			f.dirResourceKeys.Store(item.Id, item.ResourceKey)
 		}
-		baseObject, err := f.newBaseObject(ctx, remote, item)
-		if err != nil {
-			return nil, err
+		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
+		d := fs.NewDir(remote, when).SetID(item.Id)
+		if len(item.Parents) > 0 {
+			d.SetParentID(item.Parents[0])
 		}
-		d := &Directory{baseObject: baseObject}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
@@ -2441,6 +2370,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 	// Define the metadata for the file we are going to create.
 	createInfo := &drive.File{
 		Name:         leaf,
+		Description:  leaf,
 		Parents:      []string{directoryID},
 		ModifiedTime: modTime.Format(timeFormatOut),
 	}
@@ -2605,59 +2535,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return err
 }
 
-// MkdirMetadata makes the directory passed in as dir.
-//
-// It shouldn't return an error if it already exists.
-//
-// If the metadata is not nil it is set.
-//
-// It returns the directory that was created.
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	var info *drive.File
-	dirID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err == fs.ErrorDirNotFound {
-		// Directory does not exist so create it
-		var leaf, parentID string
-		leaf, parentID, err = f.dirCache.FindPath(ctx, dir, true)
-		if err != nil {
-			return nil, err
-		}
-		info, err = f.createDir(ctx, parentID, leaf, metadata)
-	} else if err == nil {
-		// Directory exists and needs updating
-		info, err = f.updateDir(ctx, dirID, metadata)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Convert the info into a directory entry
-	entry, err := f.itemToDirEntry(ctx, dir, info)
-	if err != nil {
-		return nil, err
-	}
-	dirEntry, ok := entry.(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be an fs.Directory", entry)
-	}
-
-	return dirEntry, nil
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	dirID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-	o := baseObject{
-		fs:     f,
-		remote: dir,
-		id:     dirID,
-	}
-	return o.SetModTime(ctx, modTime)
-}
-
 // delete a file or directory unconditionally by ID
 func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 	return f.pacer.Call(func() (bool, error) {
@@ -2694,7 +2571,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	if shortcutID != "" {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
-	trashedFiles := false
+	var trashedFiles = false
 	if check {
 		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
 			if !item.Trashed {
@@ -2801,12 +2678,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		createInfo.Description = ""
 	}
 
-	// Adjust metadata if required
-	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), createInfo, false)
-	if err != nil {
-		return nil, err
-	}
-
 	// get the ID of the thing to copy
 	// copy the contents if CopyShortcutContent
 	// else copy the shortcut only
@@ -2820,7 +2691,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		copy := f.svc.Files.Copy(id, createInfo).
-			Fields(f.getFileFields(ctx)).
+			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever)
 		srcObj.addResourceKey(copy.Header())
@@ -2840,7 +2711,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// FIXME remove this when google fixes the problem!
 	if isDoc {
 		// A short sleep is needed here in order to make the
-		// change effective, without it is ignored. This is
+		// change effective, without it is is ignored. This is
 		// probably some eventual consistency nastiness.
 		sleepTime := 2 * time.Second
 		fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
@@ -2856,11 +2727,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
 		}
 	}
-	// Finalise metadata
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
 	return newObject, nil
 }
 
@@ -2931,6 +2797,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
+
 	if err != nil {
 		return err
 	}
@@ -3033,19 +2900,13 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
-	// Adjust metadata if required
-	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), dstInfo, true)
-	if err != nil {
-		return nil, err
-	}
-
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Update(shortcutID(srcObj.id), dstInfo).
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
-			Fields(f.getFileFields(ctx)).
+			Fields(partialFields).
 			SupportsAllDrives(true).
 			Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
@@ -3054,11 +2915,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	// Finalise metadata
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
 	return f.newObjectWithInfo(ctx, remote, info)
 }
 
@@ -3191,7 +3047,6 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		}
 	}()
 }
-
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
@@ -3565,94 +3420,6 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	return nil
 }
 
-// Run the drive query calling fn on each entry found
-func (f *Fs) queryFn(ctx context.Context, query string, fn func(*drive.File)) (err error) {
-	list := f.svc.Files.List()
-	if query != "" {
-		list.Q(query)
-	}
-
-	if f.opt.ListChunk > 0 {
-		list.PageSize(f.opt.ListChunk)
-	}
-	list.SupportsAllDrives(true)
-	list.IncludeItemsFromAllDrives(true)
-	if f.isTeamDrive && !f.opt.SharedWithMe {
-		list.DriveId(f.opt.TeamDriveID)
-		list.Corpora("drive")
-	}
-	// If using appDataFolder then need to add Spaces
-	if f.rootFolderID == "appDataFolder" {
-		list.Spaces("appDataFolder")
-	}
-	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
-	for {
-		var files *drive.FileList
-		err = f.pacer.Call(func() (bool, error) {
-			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			return fmt.Errorf("failed to execute query: %w", err)
-		}
-		if files.IncompleteSearch {
-			fs.Errorf(f, "search result INCOMPLETE")
-		}
-		for _, item := range files.Files {
-			fn(item)
-		}
-		if files.NextPageToken == "" {
-			break
-		}
-		list.PageToken(files.NextPageToken)
-	}
-	return nil
-}
-
-// Run the drive query returning the entries found
-func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
-	var results []*drive.File
-	err = f.queryFn(ctx, query, func(item *drive.File) {
-		results = append(results, item)
-	})
-	if err != nil {
-		return nil, err
-	}
-	return results, nil
-}
-
-// Rescue, list or delete orphaned files
-func (f *Fs) rescue(ctx context.Context, dirID string, delete bool) (err error) {
-	return f.queryFn(ctx, "'me' in owners and trashed=false", func(item *drive.File) {
-		if len(item.Parents) != 0 {
-			return
-		}
-		// Have found an orphaned entry
-		if delete {
-			fs.Infof(item.Name, "Deleting orphan %q into trash", item.Id)
-			err = f.delete(ctx, item.Id, true)
-			if err != nil {
-				fs.Errorf(item.Name, "Failed to delete orphan %q: %v", item.Id, err)
-			}
-		} else if dirID == "" {
-			operations.SyncPrintf("%q, %q\n", item.Name, item.Id)
-		} else {
-			fs.Infof(item.Name, "Rescuing orphan %q", item.Id)
-			err = f.pacer.Call(func() (bool, error) {
-				_, err = f.svc.Files.Update(item.Id, nil).
-					AddParents(dirID).
-					Fields(f.getFileFields(ctx)).
-					SupportsAllDrives(true).
-					Context(ctx).Do()
-				return f.shouldRetry(ctx, err)
-			})
-			if err != nil {
-				fs.Errorf(item.Name, "Failed to rescue orphan %q: %v", item.Id, err)
-			}
-		}
-	})
-}
-
 var commandHelp = []fs.CommandHelp{{
 	Name:  "get",
 	Short: "Get command for fetching the drive config parameters",
@@ -3803,78 +3570,6 @@ Use the --interactive/-i or --dry-run flag to see what would be copied before co
 }, {
 	Name:  "importformats",
 	Short: "Dump the import formats for debug purposes",
-}, {
-	Name:  "query",
-	Short: "List files using Google Drive query language",
-	Long: `This command lists files based on a query
-
-Usage:
-
-    rclone backend query drive: query
-    
-The query syntax is documented at [Google Drive Search query terms and 
-operators](https://developers.google.com/drive/api/guides/ref-search-terms).
-
-For example:
-
-	rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
-
-If the query contains literal ' or \ characters, these need to be escaped with
-\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a 
-file named "foo ' \.txt":
-
-	rclone backend query drive: "name = 'foo \' \\\.txt'"
-
-The result is a JSON array of matches, for example:
-
-    [
-	{
-		"createdTime": "2017-06-29T19:58:28.537Z",
-		"id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
-		"md5Checksum": "68518d16be0c6fbfab918be61d658032",
-		"mimeType": "text/plain",
-		"modifiedTime": "2024-02-02T10:40:02.874Z",
-		"name": "foo ' \\.txt",
-		"parents": [
-			"0BxAe_BCDE4zkFGZpcWJGek0xbzC"
-		],
-		"resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
-		"sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
-		"size": "311",
-		"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
-	}
-    ]`,
-}, {
-	Name:  "rescue",
-	Short: "Rescue or delete any orphaned files",
-	Long: `This command rescues or deletes any orphaned files or directories.
-
-Sometimes files can get orphaned in Google Drive. This means that they
-are no longer in any folder in Google Drive.
-
-This command finds those files and either rescues them to a directory
-you specify or deletes them.
-
-Usage:
-
-This can be used in 3 ways.
-
-First, list all orphaned files
-
-    rclone backend rescue drive:
-
-Second rescue all orphaned files to the directory indicated
-
-    rclone backend rescue drive: "relative/path/to/rescue/directory"
-
-e.g. To rescue all orphans to a directory called "Orphans" in the top level
-
-    rclone backend rescue drive: Orphans
-
-Third delete all orphaned files to the trash
-
-    rclone backend rescue drive: -o delete
-`,
 }}
 
 // Command the backend to run a named command
@@ -3992,32 +3687,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		return f.exportFormats(ctx), nil
 	case "importformats":
 		return f.importFormats(ctx), nil
-	case "query":
-		if len(arg) == 1 {
-			query := arg[0]
-			results, err := f.query(ctx, query)
-			if err != nil {
-				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
-			}
-			return results, nil
-		}
-		return nil, errors.New("need a query argument")
-	case "rescue":
-		dirID := ""
-		_, delete := opt["delete"]
-		if len(arg) == 0 {
-			// no arguments - list only
-		} else if !delete && len(arg) == 1 {
-			dir := arg[0]
-			dirID, err = f.dirCache.FindDir(ctx, dir, true)
-			if err != nil {
-				return nil, fmt.Errorf("failed to find or create rescue directory %q: %w", dir, err)
-			}
-			fs.Infof(f, "Rescuing orphans into %q", dir)
-		} else {
-			return nil, errors.New("syntax error: need 0 or 1 args or -o delete")
-		}
-		return nil, f.rescue(ctx, dirID, delete)
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -4061,9 +3730,8 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	}
 	return "", hash.ErrUnsupported
 }
-
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
+	if t != hash.MD5 {
 		return "", hash.ErrUnsupported
 	}
 	return "", nil
@@ -4076,8 +3744,7 @@ func (o *baseObject) Size() int64 {
 
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
-	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error,
-) {
+	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -4290,13 +3957,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	return o.baseObject.open(ctx, o.url, options...)
 }
-
 func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
 	var offset, end int64 = 0, -1
-	newOptions := options[:0]
+	var newOptions = options[:0]
 	for _, o := range options {
 		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
@@ -4323,10 +3989,9 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
 	}
 	return
 }
-
 func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
-	data := o.content
+	var data = o.content
 	for _, option := range options {
 		switch x := option.(type) {
 		case *fs.SeekOption:
@@ -4351,8 +4016,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
-	src fs.ObjectInfo,
-) (info *drive.File, err error) {
+	src fs.ObjectInfo) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
 	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
@@ -4430,7 +4094,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	return nil
 }
-
 func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	srcMimeType := fs.MimeType(ctx, src)
 	importMimeType := ""
@@ -4526,42 +4189,10 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
-
 func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
 
-// Items returns the count of items in this directory or this
-// directory and subdirectories if known, -1 for unknown
-func (d *Directory) Items() int64 {
-	return -1
-}
-
-// SetMetadata sets metadata for a Directory
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	info, err := d.fs.updateDir(ctx, d.id, metadata)
-	if err != nil {
-		return fmt.Errorf("failed to update directory info: %w", err)
-	}
-	// Update directory from info returned
-	baseObject, err := d.fs.newBaseObject(ctx, d.remote, info)
-	if err != nil {
-		return fmt.Errorf("failed to process directory info: %w", err)
-	}
-	d.baseObject = baseObject
-	return err
-}
-
-// Hash does nothing on a directory
-//
-// This method is implemented with the incorrect type signature to
-// stop the Directory type asserting to fs.Object or fs.ObjectInfo
-func (d *Directory) Hash() {
-	// Does nothing
-}
-
 // templates for document link files
 const (
 	urlTemplate = `[InternetShortcut]{{"\r"}}
@@ -4611,8 +4242,6 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
@@ -4627,8 +4256,4 @@ var (
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
 	_ fs.ParentIDer      = (*linkObject)(nil)
-	_ fs.Directory       = (*Directory)(nil)
-	_ fs.SetModTimer     = (*Directory)(nil)
-	_ fs.SetMetadataer   = (*Directory)(nil)
-	_ fs.ParentIDer      = (*Directory)(nil)
 )

backend/drive/drive_internal_test.go

@@ -95,7 +95,7 @@ func TestInternalParseExtensions(t *testing.T) {
 		wantErr error
 	}{
 		{"doc", []string{".doc"}, nil},
-		{" docx ,XLSX, 	pptx,svg,md", []string{".docx", ".xlsx", ".pptx", ".svg", ".md"}, nil},
+		{" docx ,XLSX, 	pptx,svg", []string{".docx", ".xlsx", ".pptx", ".svg"}, nil},
 		{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
 		{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
 	} {
@@ -524,49 +524,12 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	})
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
-func (f *Fs) InternalTestQuery(t *testing.T) {
-	ctx := context.Background()
-	var err error
-	t.Run("BadQuery", func(t *testing.T) {
-		_, err = f.query(ctx, "this is a bad query")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "failed to execute query")
-	})
-
-	t.Run("NoMatch", func(t *testing.T) {
-		results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
-		require.NoError(t, err)
-		assert.Len(t, results, 0)
-	})
-
-	t.Run("GoodQuery", func(t *testing.T) {
-		pathSegments := strings.Split(existingFile, "/")
-		var parent string
-		for _, item := range pathSegments {
-			// the file name contains ' characters which must be escaped
-			escapedItem := f.opt.Enc.FromStandardName(item)
-			escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
-			escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
-
-			results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
-			require.NoError(t, err)
-			require.True(t, len(results) > 0)
-			for _, result := range results {
-				assert.True(t, len(result.Id) > 0)
-				assert.Equal(t, result.Name, item)
-			}
-			parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
-		}
-	})
-}
-
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
 	// Check set up for filtering
 	assert.True(t, f.Features().FilterAware)
 
-	opt := &filter.Options{}
+	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
 	flt, err := filter.NewFilter(opt)
@@ -648,7 +611,6 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
-	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }

backend/drive/metadata.go

@@ -9,8 +9,6 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/errcount"
 	"golang.org/x/sync/errgroup"
 	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
@@ -39,7 +37,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		Example: "true",
 	},
 	"writers-can-share": {
-		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives.",
+		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated for items in shared drives.",
 		Type:    "boolean",
 		Example: "false",
 	},
@@ -137,30 +135,23 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
 
 // Set the permissions on the info
 func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
-	errs := errcount.New()
 	for _, perm := range permissions {
 		if perm.Role == "owner" {
 			// ignore owner permissions - these are set with owner
 			continue
 		}
 		cleanPermissionForWrite(perm)
-		err := f.pacer.Call(func() (bool, error) {
-			_, err := f.svc.Permissions.Create(info.Id, perm).
+		err = f.pacer.Call(func() (bool, error) {
+			_, err = f.svc.Permissions.Create(info.Id, perm).
 				SupportsAllDrives(true).
-				SendNotificationEmail(false).
 				Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
-			fs.Errorf(f, "Failed to set permission %s for %q: %v", perm.Role, perm.EmailAddress, err)
-			errs.Add(err)
+			return fmt.Errorf("failed to set permission: %w", err)
 		}
 	}
-	err = errs.Err("failed to set permission")
-	if err != nil {
-		err = fserrors.NoRetryError(err)
-	}
-	return err
+	return nil
 }
 
 // Clean attributes from permissions which we can't write
@@ -262,7 +253,7 @@ func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.La
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return fmt.Errorf("failed to set labels: %w", err)
+		return fmt.Errorf("failed to set owner: %w", err)
 	}
 	return nil
 }
@@ -372,7 +363,6 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
 		// shared drives.
 		if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
 			// Don't process permissions if there aren't any specifically set
-			fs.Debugf(o, "Ignoring %d permissions and %d permissionIds as is shared drive with hasAugmentedPermissions false", len(info.Permissions), len(info.PermissionIds))
 			info.Permissions = nil
 			info.PermissionIds = nil
 		}
@@ -537,12 +527,8 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 				return nil, err
 			}
 		case "writers-can-share":
-			if !f.isTeamDrive {
-				if err := parseBool(&updateInfo.WritersCanShare); err != nil {
-					return nil, err
-				}
-			} else {
-				fs.Debugf(f, "Ignoring %s=%s as can't set on shared drives", k, v)
+			if err := parseBool(&updateInfo.WritersCanShare); err != nil {
+				return nil, err
 			}
 		case "viewed-by-me":
 			// Can't write this
@@ -554,12 +540,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Owner on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setOwner(ctx, info, v)
-				if err != nil && f.opt.MetadataOwner.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setOwner(ctx, info, v)
 			})
 		case "permissions":
 			if !f.opt.MetadataPermissions.IsSet(rwWrite) {
@@ -572,13 +553,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Permissions on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setPermissions(ctx, info, perms)
-				if err != nil && f.opt.MetadataPermissions.IsSet(rwFailOK) {
-					// We've already logged the permissions errors individually here
-					fs.Debugf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setPermissions(ctx, info, perms)
 			})
 		case "labels":
 			if !f.opt.MetadataLabels.IsSet(rwWrite) {
@@ -591,12 +566,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Labels on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setLabels(ctx, info, labels)
-				if err != nil && f.opt.MetadataLabels.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setLabels(ctx, info, labels)
 			})
 		case "folder-color-rgb":
 			updateInfo.FolderColorRgb = v

backend/dropbox/batcher.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/rclone/rclone/fs/fserrors"
 )
 
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
@@ -20,10 +21,14 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
 		}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/dropbox/dropbox.go

@@ -47,7 +47,6 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -94,7 +93,7 @@ const (
 
 var (
 	// Description of how to auth for this app
-	dropboxConfig = &oauthutil.Config{
+	dropboxConfig = &oauth2.Config{
 		Scopes: []string{
 			"files.metadata.write",
 			"files.content.write",
@@ -109,8 +108,7 @@ var (
 		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
 		// 	TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
 		// },
-		AuthURL:      dropbox.OAuthEndpoint("").AuthURL,
-		TokenURL:     dropbox.OAuthEndpoint("").TokenURL,
+		Endpoint:     dropbox.OAuthEndpoint(""),
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -135,7 +133,7 @@ var (
 )
 
 // Gets an oauth config with the right scopes
-func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
+func getOauthConfig(m configmap.Mapper) *oauth2.Config {
 	// If not impersonating, use standard scopes
 	if impersonate, _ := m.Get("impersonate"); impersonate == "" {
 		return dropboxConfig
@@ -218,10 +216,7 @@ are supported.
 
 Note that we don't unmount the shared folder afterwards so the 
 --dropbox-shared-folders can be omitted after the first use of a particular 
-shared folder.
-
-See also --dropbox-root-namespace for an alternative way to work with shared
-folders.`,
+shared folder.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -242,11 +237,6 @@ folders.`,
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8,
-		}, {
-			Name:     "root_namespace",
-			Help:     "Specify a different Dropbox namespace ID to use as the root for all paths.",
-			Default:  "",
-			Advanced: true,
 		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
 	})
 }
@@ -263,7 +253,6 @@ type Options struct {
 	AsyncBatch    bool                 `config:"async_batch"`
 	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
 	Enc           encoder.MultiEncoder `config:"encoding"`
-	RootNsid      string               `config:"root_namespace"`
 }
 
 // Fs represents a remote dropbox server
@@ -318,46 +307,32 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Some specific errors which should be excluded from retries
-func shouldRetryExclude(ctx context.Context, err error) (bool, error) {
-	if err == nil {
-		return false, err
-	}
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	// First check for specific errors
-	//
-	// These come back from the SDK in a whole host of different
-	// error types, but there doesn't seem to be a consistent way
-	// of reading the error cause, so here we just check using the
-	// error string which isn't perfect but does the job.
+	if err == nil {
+		return false, err
+	}
 	errString := err.Error()
+	// First check for specific errors
 	if strings.Contains(errString, "insufficient_space") {
 		return false, fserrors.FatalError(err)
 	} else if strings.Contains(errString, "malformed_path") {
 		return false, fserrors.NoRetryError(err)
 	}
-	return true, err
-}
-
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if retry, err := shouldRetryExclude(ctx, err); !retry {
-		return retry, err
-	}
 	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
-			fs.Logf(nil, "Error %v. Too many requests or write operations. Trying again in %d seconds.", err, e.RateLimitError.RetryAfter)
+			fs.Logf(errString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
 			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
 	// Keep old behavior for backward compatibility
-	errString := err.Error()
 	if strings.Contains(errString, "too_many_write_operations") || strings.Contains(errString, "too_many_requests") || errString == "" {
 		return true, err
 	}
@@ -402,7 +377,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	oldToken = strings.TrimSpace(oldToken)
 	if ok && oldToken != "" && oldToken[0] != '{' {
 		fs.Infof(name, "Converting token to new format")
-		newToken := fmt.Sprintf(`{"access_token":%q,"token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
+		newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
 		err := config.SetValueAndSave(name, config.ConfigToken, newToken)
 		if err != nil {
 			return nil, fmt.Errorf("NewFS convert token: %w", err)
@@ -453,15 +428,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		members := []*team.UserSelectorArg{&user}
 		args := team.NewMembersGetInfoArgs(members)
 
-		memberIDs, err := f.team.MembersGetInfo(args)
+		memberIds, err := f.team.MembersGetInfo(args)
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
-		if len(memberIDs) == 0 || memberIDs[0].MemberInfo == nil || memberIDs[0].MemberInfo.Profile == nil {
+		if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil {
 			return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate)
 		}
 
-		cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
+		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
 
 	f.srv = files.New(cfg)
@@ -527,11 +502,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	f.features.Fill(ctx, f)
 
-	if f.opt.RootNsid != "" {
-		f.ns = f.opt.RootNsid
-		fs.Debugf(f, "Overriding root namespace to %q", f.ns)
-	} else if strings.HasPrefix(root, "/") {
-		// If root starts with / then use the actual root
+	// If root starts with / then use the actual root
+	if strings.HasPrefix(root, "/") {
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
 			acc, err = f.users.GetCurrentAccount()
@@ -672,7 +644,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
-// listSharedFolders lists all available shared folders mounted and not mounted
+// listSharedFoldersApi lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
 func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
@@ -1036,20 +1008,13 @@ func (f *Fs) Precision() time.Duration {
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
 
-	// Find and remove existing object
-	cleanup, err := operations.RemoveExisting(ctx, f, remote, "server side copy")
-	if err != nil {
-		return nil, err
-	}
-	defer cleanup(&err)
-
 	// Temporary Object under construction
 	dstObj := &Object{
 		fs:     f,
@@ -1063,6 +1028,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
 		},
 	}
+	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
 		result, err = f.srv.CopyV2(&arg)
@@ -1265,7 +1231,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return nil, err
 	}
 	var total uint64
-	used := q.Used
+	var used = q.Used
 	if q.Allocation != nil {
 		if q.Allocation.Individual != nil {
 			total += q.Allocation.Individual.Allocated
@@ -1714,10 +1680,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
-		if retry, err := shouldRetryExclude(ctx, err); !retry {
-			return retry, err
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
 		}
-		// after the first chunk is uploaded, we retry everything except the excluded errors
+		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
 	if err != nil {

backend/fichier/api.go

@@ -61,7 +61,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 			return false, err // No such user
 		case 186:
 			return false, err // IP blocked?
-		case 374, 412: // Flood detected seems to be #412 now
+		case 374:
 			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
 			time.Sleep(30 * time.Second)
 		default:

backend/fichier/fichier.go

@@ -441,28 +441,23 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(src, "Can't move - not same remote type")
 		return nil, fs.ErrorCantMove
 	}
-	srcFs := srcObj.fs
 
 	// Find current directory ID
-	srcLeaf, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcObj.remote, false)
+	_, currentDirectoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		return nil, err
 	}
 
 	// Create temporary object
-	dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote)
+	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
 
 	// If it is in the correct directory, just rename it
 	var url string
-	if srcDirectoryID == dstDirectoryID {
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			return src, nil
-		}
-		resp, err := f.renameFile(ctx, srcObj.file.URL, dstLeaf)
+	if currentDirectoryID == directoryID {
+		resp, err := f.renameFile(ctx, srcObj.file.URL, leaf)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't rename file: %w", err)
 		}
@@ -471,16 +466,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		}
 		url = resp.URLs[0].URL
 	} else {
-		dstFolderID, err := strconv.Atoi(dstDirectoryID)
+		folderID, err := strconv.Atoi(directoryID)
 		if err != nil {
 			return nil, err
 		}
-		rename := dstLeaf
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			rename = ""
-		}
-		resp, err := f.moveFile(ctx, srcObj.file.URL, dstFolderID, rename)
+		resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't move file: %w", err)
 		}

backend/filescom/filescom.go

@@ -1,901 +0,0 @@
-// Package filescom provides an interface to the Files.com
-// object storage system.
-package filescom
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-	"time"
-
-	files_sdk "github.com/Files-com/files-sdk-go/v3"
-	"github.com/Files-com/files-sdk-go/v3/bundle"
-	"github.com/Files-com/files-sdk-go/v3/file"
-	file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
-	"github.com/Files-com/files-sdk-go/v3/folder"
-	"github.com/Files-com/files-sdk-go/v3/session"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-)
-
-/*
-Run of rclone info
-stringNeedsEscaping = []rune{
-        '/', '\x00'
-}
-maxFileLength = 512 // for 1 byte unicode characters
-maxFileLength = 512 // for 2 byte unicode characters
-maxFileLength = 512 // for 3 byte unicode characters
-maxFileLength = 512 // for 4 byte unicode characters
-canWriteUnnormalized = true
-canReadUnnormalized   = true
-canReadRenormalized   = true
-canStream = true
-*/
-
-const (
-	minSleep      = 10 * time.Millisecond
-	maxSleep      = 2 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
-
-	folderNotEmpty = "processing-failure/folder-not-empty"
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "filescom",
-		Description: "Files.com",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name: "site",
-				Help: "Your site subdomain (e.g. mysite) or custom domain (e.g. myfiles.customdomain.com).",
-			}, {
-				Name: "username",
-				Help: "The username used to authenticate with Files.com.",
-			}, {
-				Name:       "password",
-				Help:       "The password used to authenticate with Files.com.",
-				IsPassword: true,
-			}, {
-				Name:      "api_key",
-				Help:      "The API key used to authenticate with Files.com.",
-				Advanced:  true,
-				Sensitive: true,
-			}, {
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Display |
-					encoder.EncodeBackSlash |
-					encoder.EncodeRightSpace |
-					encoder.EncodeRightCrLfHtVt |
-					encoder.EncodeInvalidUtf8),
-			}},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	Site     string               `config:"site"`
-	Username string               `config:"username"`
-	Password string               `config:"password"`
-	APIKey   string               `config:"api_key"`
-	Enc      encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs represents a remote files.com server
-type Fs struct {
-	name            string                 // name of this remote
-	root            string                 // the path we are working on
-	opt             Options                // parsed options
-	features        *fs.Features           // optional features
-	fileClient      *file.Client           // the connection to the file API
-	folderClient    *folder.Client         // the connection to the folder API
-	migrationClient *file_migration.Client // the connection to the file migration API
-	bundleClient    *bundle.Client         // the connection to the bundle API
-	pacer           *fs.Pacer              // pacer for API calls
-}
-
-// Object describes a files object
-//
-// Will definitely have info but maybe not meta
-type Object struct {
-	fs       *Fs       // what this object is part of
-	remote   string    // The remote path
-	size     int64     // size of the object
-	crc32    string    // CRC32 of the object content
-	md5      string    // MD5 of the object content
-	mimeType string    // Content-Type of the object
-	modTime  time.Time // modification time of the object
-}
-
-// ------------------------------------------------------------
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("files root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Encode remote and turn it into an absolute path in the share
-func (f *Fs) absPath(remote string) string {
-	return f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-
-	if apiErr, ok := err.(files_sdk.ResponseError); ok {
-		for _, e := range retryErrorCodes {
-			if apiErr.HttpCode == e {
-				fs.Debugf(nil, "Retrying API error %v", err)
-				return true, err
-			}
-		}
-	}
-
-	return fserrors.ShouldRetry(err), err
-}
-
-// readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *files_sdk.File, err error) {
-	params := files_sdk.FileFindParams{
-		Path: f.absPath(path),
-	}
-
-	var file files_sdk.File
-	err = f.pacer.Call(func() (bool, error) {
-		file, err = f.fileClient.Find(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &file, nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	// Parse config into Options struct
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	root = strings.Trim(root, "/")
-
-	config, err := newClientConfig(ctx, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	f := &Fs{
-		name:            name,
-		root:            root,
-		opt:             *opt,
-		fileClient:      &file.Client{Config: config},
-		folderClient:    &folder.Client{Config: config},
-		migrationClient: &file_migration.Client{Config: config},
-		bundleClient:    &bundle.Client{Config: config},
-		pacer:           fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-	}
-	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		CanHaveEmptyDirectories:  true,
-		ReadMimeType:             true,
-		DirModTimeUpdatesOnWrite: true,
-	}).Fill(ctx, f)
-
-	if f.root != "" {
-		info, err := f.readMetaDataForPath(ctx, "")
-		if err == nil && !info.IsDir() {
-			f.root = path.Dir(f.root)
-			if f.root == "." {
-				f.root = ""
-			}
-			return f, fs.ErrorIsFile
-		}
-	}
-
-	return f, err
-}
-
-func newClientConfig(ctx context.Context, opt *Options) (config files_sdk.Config, err error) {
-	if opt.Site != "" {
-		if strings.Contains(opt.Site, ".") {
-			config.EndpointOverride = opt.Site
-		} else {
-			config.Subdomain = opt.Site
-		}
-
-		_, err = url.ParseRequestURI(config.Endpoint())
-		if err != nil {
-			err = fmt.Errorf("invalid domain or subdomain: %v", opt.Site)
-			return
-		}
-	}
-
-	config = config.Init().SetCustomClient(fshttp.NewClient(ctx))
-
-	if opt.APIKey != "" {
-		config.APIKey = opt.APIKey
-		return
-	}
-
-	if opt.Username == "" {
-		err = errors.New("username not found")
-		return
-	}
-	if opt.Password == "" {
-		err = errors.New("password not found")
-		return
-	}
-	opt.Password, err = obscure.Reveal(opt.Password)
-	if err != nil {
-		return
-	}
-
-	sessionClient := session.Client{Config: config}
-	params := files_sdk.SessionCreateParams{
-		Username: opt.Username,
-		Password: opt.Password,
-	}
-
-	thisSession, err := sessionClient.Create(params, files_sdk.WithContext(ctx))
-	if err != nil {
-		err = fmt.Errorf("couldn't create session: %w", err)
-		return
-	}
-
-	config.SessionId = thisSession.Id
-	return
-}
-
-// Return an Object from a path
-//
-// If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *files_sdk.File) (fs.Object, error) {
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-	var err error
-	if file != nil {
-		err = o.setMetaData(file)
-	} else {
-		err = o.readMetaData(ctx) // reads info and meta, returning an error
-	}
-	if err != nil {
-		return nil, err
-	}
-	return o, nil
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(ctx, remote, nil)
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	var it *folder.Iter
-	params := files_sdk.FolderListForParams{
-		Path: f.absPath(dir),
-	}
-
-	err = f.pacer.Call(func() (bool, error) {
-		it, err = f.folderClient.ListFor(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list files: %w", err)
-	}
-
-	for it.Next() {
-		item := ptr(it.File())
-		remote := f.opt.Enc.ToStandardPath(item.DisplayName)
-		remote = path.Join(dir, remote)
-		if remote == dir {
-			continue
-		}
-
-		if item.IsDir() {
-			d := fs.NewDir(remote, item.ModTime())
-			entries = append(entries, d)
-		} else {
-			o, err := f.newObjectWithInfo(ctx, remote, item)
-			if err != nil {
-				return nil, err
-			}
-			entries = append(entries, o)
-		}
-	}
-	err = it.Err()
-	if files_sdk.IsNotExist(err) {
-		return nil, fs.ErrorDirNotFound
-	}
-	return
-}
-
-// Creates from the parameters passed in a half finished Object which
-// must have setMetaData called on it
-//
-// Returns the object and error.
-//
-// Used to create new objects
-func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, err error) {
-	// Create the directory for the object if it doesn't exist
-	err = f.mkParentDir(ctx, remote)
-	if err != nil {
-		return
-	}
-	// Temporary Object under construction
-	o = &Object{
-		fs:     f,
-		remote: remote,
-	}
-	return o, nil
-}
-
-// Put the object
-//
-// Copy the reader in to the new object which is returned.
-//
-// The new object may have been created if an error is returned
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	// Temporary Object under construction
-	fs := &Object{
-		fs:     f,
-		remote: src.Remote(),
-	}
-	return fs, fs.Update(ctx, in, src, options...)
-}
-
-// PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(ctx, in, src, options...)
-}
-
-func (f *Fs) mkdir(ctx context.Context, path string) error {
-	if path == "" || path == "." {
-		return nil
-	}
-
-	params := files_sdk.FolderCreateParams{
-		Path:         path,
-		MkdirParents: ptr(true),
-	}
-
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.folderClient.Create(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if files_sdk.IsExist(err) {
-		return nil
-	}
-	return err
-}
-
-// Make the parent directory of remote
-func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
-	return f.mkdir(ctx, path.Dir(f.absPath(remote)))
-}
-
-// Mkdir creates the container if it doesn't exist
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	return f.mkdir(ctx, f.absPath(dir))
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	o := Object{
-		fs:     f,
-		remote: dir,
-	}
-	return o.SetModTime(ctx, modTime)
-}
-
-// purgeCheck removes the root directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
-	path := f.absPath(dir)
-	if path == "" || path == "." {
-		return errors.New("can't purge root directory")
-	}
-
-	params := files_sdk.FileDeleteParams{
-		Path:      path,
-		Recursive: ptr(!check),
-	}
-
-	err := f.pacer.Call(func() (bool, error) {
-		err := f.fileClient.Delete(params, files_sdk.WithContext(ctx))
-		// Allow for eventual consistency deletion of child objects.
-		if isFolderNotEmpty(err) {
-			return true, err
-		}
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		if files_sdk.IsNotExist(err) {
-			return fs.ErrorDirNotFound
-		} else if isFolderNotEmpty(err) {
-			return fs.ErrorDirectoryNotEmpty
-		}
-
-		return fmt.Errorf("rmdir failed: %w", err)
-	}
-	return nil
-}
-
-// Rmdir deletes the root folder
-//
-// Returns an error if it isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
-}
-
-// Precision return the precision of this Fs
-func (f *Fs) Precision() time.Duration {
-	return time.Second
-}
-
-// Copy src to this remote using server-side copy operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.Object, err error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't copy - not same remote type")
-		return nil, fs.ErrorCantCopy
-	}
-	err = srcObj.readMetaData(ctx)
-	if err != nil {
-		return
-	}
-
-	srcPath := srcObj.fs.absPath(srcObj.remote)
-	dstPath := f.absPath(remote)
-	if strings.EqualFold(srcPath, dstPath) {
-		return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
-	}
-
-	// Create temporary object
-	dstObj, err = f.createObject(ctx, remote)
-	if err != nil {
-		return
-	}
-
-	// Copy the object
-	params := files_sdk.FileCopyParams{
-		Path:        srcPath,
-		Destination: dstPath,
-		Overwrite:   ptr(true),
-	}
-
-	var action files_sdk.FileAction
-	err = f.pacer.Call(func() (bool, error) {
-		action, err = f.fileClient.Copy(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return
-	}
-
-	err = f.waitForAction(ctx, action, "copy")
-	if err != nil {
-		return
-	}
-
-	err = dstObj.SetModTime(ctx, srcObj.modTime)
-	return
-}
-
-// Purge deletes all the files and the container
-//
-// Optional interface: Only implement this if you have a way of
-// deleting all the files quicker than just running Remove() on the
-// result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
-}
-
-// move a file or folder
-func (f *Fs) move(ctx context.Context, src *Fs, srcRemote string, dstRemote string) (info *files_sdk.File, err error) {
-	// Move the object
-	params := files_sdk.FileMoveParams{
-		Path:        src.absPath(srcRemote),
-		Destination: f.absPath(dstRemote),
-	}
-
-	var action files_sdk.FileAction
-	err = f.pacer.Call(func() (bool, error) {
-		action, err = f.fileClient.Move(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	err = f.waitForAction(ctx, action, "move")
-	if err != nil {
-		return nil, err
-	}
-
-	info, err = f.readMetaDataForPath(ctx, dstRemote)
-	return
-}
-
-func (f *Fs) waitForAction(ctx context.Context, action files_sdk.FileAction, operation string) (err error) {
-	var migration files_sdk.FileMigration
-	err = f.pacer.Call(func() (bool, error) {
-		migration, err = f.migrationClient.Wait(action, func(migration files_sdk.FileMigration) {
-			// noop
-		}, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err == nil && migration.Status != "completed" {
-		return fmt.Errorf("%v did not complete successfully: %v", operation, migration.Status)
-	}
-	return
-}
-
-// Move src to this remote using server-side move operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantMove
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	// Do the move
-	info, err := f.move(ctx, srcObj.fs, srcObj.remote, dstObj.remote)
-	if err != nil {
-		return nil, err
-	}
-
-	err = dstObj.setMetaData(info)
-	if err != nil {
-		return nil, err
-	}
-	return dstObj, nil
-}
-
-// DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantDirMove
-//
-// If destination exists then return fs.ErrorDirExists
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
-	srcFs, ok := src.(*Fs)
-	if !ok {
-		fs.Debugf(srcFs, "Can't move directory - not same remote type")
-		return fs.ErrorCantDirMove
-	}
-
-	// Check if destination exists
-	_, err = f.readMetaDataForPath(ctx, dstRemote)
-	if err == nil {
-		return fs.ErrorDirExists
-	}
-
-	// Create temporary object
-	dstObj, err := f.createObject(ctx, dstRemote)
-	if err != nil {
-		return
-	}
-
-	// Do the move
-	_, err = f.move(ctx, srcFs, srcRemote, dstObj.remote)
-	return
-}
-
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (url string, err error) {
-	params := files_sdk.BundleCreateParams{
-		Paths: []string{f.absPath(remote)},
-	}
-	if expire < fs.DurationOff {
-		params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
-	}
-
-	var bundle files_sdk.Bundle
-	err = f.pacer.Call(func() (bool, error) {
-		bundle, err = f.bundleClient.Create(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-
-	url = bundle.Url
-	return
-}
-
-// Hashes returns the supported hash sets.
-func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet(hash.CRC32, hash.MD5)
-}
-
-// ------------------------------------------------------------
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// Hash returns the MD5 of an object returning a lowercase hex string
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	switch t {
-	case hash.CRC32:
-		if o.crc32 == "" {
-			return "", nil
-		}
-		return fmt.Sprintf("%08s", o.crc32), nil
-	case hash.MD5:
-		return o.md5, nil
-	}
-	return "", hash.ErrUnsupported
-}
-
-// Size returns the size of an object in bytes
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// setMetaData sets the metadata from info
-func (o *Object) setMetaData(file *files_sdk.File) error {
-	o.modTime = file.ModTime()
-
-	if !file.IsDir() {
-		o.size = file.Size
-		o.crc32 = file.Crc32
-		o.md5 = file.Md5
-		o.mimeType = file.MimeType
-	}
-
-	return nil
-}
-
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// it also sets the info
-func (o *Object) readMetaData(ctx context.Context) (err error) {
-	file, err := o.fs.readMetaDataForPath(ctx, o.remote)
-	if err != nil {
-		if files_sdk.IsNotExist(err) {
-			return fs.ErrorObjectNotFound
-		}
-		return err
-	}
-	if file.IsDir() {
-		return fs.ErrorIsDir
-	}
-	return o.setMetaData(file)
-}
-
-// ModTime returns the modification time of the object
-//
-// It attempts to read the objects mtime and if that isn't present the
-// LastModified returned in the http headers
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
-	params := files_sdk.FileUpdateParams{
-		Path:          o.fs.absPath(o.remote),
-		ProvidedMtime: &modTime,
-	}
-
-	var file files_sdk.File
-	err = o.fs.pacer.Call(func() (bool, error) {
-		file, err = o.fs.fileClient.Update(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.setMetaData(&file)
-}
-
-// Storable returns a boolean showing whether this object storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	// Offset and Count for range download
-	var offset, count int64
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size - offset
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-
-	params := files_sdk.FileDownloadParams{
-		Path: o.fs.absPath(o.remote),
-	}
-
-	headers := &http.Header{}
-	headers.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, offset+count-1))
-	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.fileClient.Download(
-			params,
-			files_sdk.WithContext(ctx),
-			files_sdk.RequestHeadersOption(headers),
-			files_sdk.ResponseBodyOption(func(closer io.ReadCloser) error {
-				in = closer
-				return err
-			}),
-		)
-		return shouldRetry(ctx, err)
-	})
-	return
-}
-
-// Returns a pointer to t - useful for returning pointers to constants
-func ptr[T any](t T) *T {
-	return &t
-}
-
-func isFolderNotEmpty(err error) bool {
-	var re files_sdk.ResponseError
-	ok := errors.As(err, &re)
-	return ok && re.Type == folderNotEmpty
-}
-
-// Update the object with the contents of the io.Reader, modTime and size
-//
-// If existing is set then it updates the object rather than creating a new one.
-//
-// The new object may have been created if an error is returned.
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	uploadOpts := []file.UploadOption{
-		file.UploadWithContext(ctx),
-		file.UploadWithReader(in),
-		file.UploadWithDestinationPath(o.fs.absPath(o.remote)),
-		file.UploadWithProvidedMtime(src.ModTime(ctx)),
-	}
-
-	err := o.fs.pacer.Call(func() (bool, error) {
-		err := o.fs.fileClient.Upload(uploadOpts...)
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return err
-	}
-
-	return o.readMetaData(ctx)
-}
-
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	params := files_sdk.FileDeleteParams{
-		Path: o.fs.absPath(o.remote),
-	}
-
-	return o.fs.pacer.Call(func() (bool, error) {
-		err := o.fs.fileClient.Delete(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-}
-
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType(ctx context.Context) string {
-	return o.mimeType
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs           = (*Fs)(nil)
-	_ fs.Purger       = (*Fs)(nil)
-	_ fs.PutStreamer  = (*Fs)(nil)
-	_ fs.Copier       = (*Fs)(nil)
-	_ fs.Mover        = (*Fs)(nil)
-	_ fs.DirMover     = (*Fs)(nil)
-	_ fs.PublicLinker = (*Fs)(nil)
-	_ fs.Object       = (*Object)(nil)
-	_ fs.MimeTyper    = (*Object)(nil)
-)

backend/filescom/filescom_test.go

@@ -1,17 +0,0 @@
-// Test Files filesystem interface
-package filescom_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/filescom"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFilesCom:",
-		NilObject:  (*filescom.Object)(nil),
-	})
-}

backend/ftp/ftp.go

@@ -85,7 +85,7 @@ to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
-			Help: strings.ReplaceAll(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
 
 Note that setting this is very likely to cause deadlocks so it should
 be used with care.
@@ -99,7 +99,7 @@ maximum of |--checkers| and |--transfers|.
 So for |concurrency 3| you'd use |--checkers 2 --transfers 2
 --check-first| or |--checkers 1 --transfers 1|.
 
-`, "|", "`"),
+`, "|", "`", -1),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -180,28 +180,12 @@ If this is set and no password is supplied then rclone will ask for a password
 			Default: "",
 			Help: `Socks 5 proxy host.
 		
-Supports the format user:pass@host:port, user@host:port, host:port.
+		Supports the format user:pass@host:port, user@host:port, host:port.
 		
-Example:
+		Example:
 		
-    myUser:myPass@localhost:9005
-`,
-			Advanced: true,
-		}, {
-			Name:    "no_check_upload",
-			Default: false,
-			Help: `Don't check the upload is OK
-
-Normally rclone will try to check the upload exists after it has
-uploaded a file to make sure the size and modification time are as
-expected.
-
-This flag stops rclone doing these checks. This enables uploading to
-folders which are write only.
-
-You will likely need to use the --inplace flag also if uploading to
-a write only folder.
-`,
+			myUser:myPass@localhost:9005
+		`,
 			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
@@ -248,7 +232,6 @@ type Options struct {
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	SocksProxy        string               `config:"socks_proxy"`
-	NoCheckUpload     bool                 `config:"no_check_upload"`
 }
 
 // Fs represents a remote FTP server
@@ -987,8 +970,6 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	f.putFtpConnection(&c, err)
 	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
-		case ftp.StatusRequestedFileActionOK: // some ftp servers apparently return 250 instead of 257
-			err = nil // see: https://forum.rclone.org/t/rclone-pop-up-an-i-o-error-when-creating-a-folder-in-a-mounted-ftp-drive/44368/
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
 		case 521: // dir already exists: error number according to RFC 959: issue #2363
@@ -1320,16 +1301,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("update stor: %w", err)
 	}
 	o.fs.putFtpConnection(&c, nil)
-	if o.fs.opt.NoCheckUpload {
-		o.info = &FileInfo{
-			Name:    o.remote,
-			Size:    uint64(src.Size()),
-			ModTime: src.ModTime(ctx),
-			precise: true,
-			IsDir:   false,
-		}
-		return nil
-	}
 	if err = o.SetModTime(ctx, src.ModTime(ctx)); err != nil {
 		return fmt.Errorf("SetModTime: %w", err)
 	}

backend/gofile/api/types.go

@@ -1,311 +0,0 @@
-// Package api has type definitions for gofile
-//
-// Converted from the API docs with help from https://mholt.github.io/json-to-go/
-package api
-
-import (
-	"fmt"
-	"time"
-)
-
-const (
-	// 2017-05-03T07:26:10-07:00
-	timeFormat = `"` + time.RFC3339 + `"`
-)
-
-// Time represents date and time information for the
-// gofile API, by using RFC3339
-type Time time.Time
-
-// MarshalJSON turns a Time into JSON (in UTC)
-func (t *Time) MarshalJSON() (out []byte, err error) {
-	timeString := (*time.Time)(t).Format(timeFormat)
-	return []byte(timeString), nil
-}
-
-// UnmarshalJSON turns JSON into a Time
-func (t *Time) UnmarshalJSON(data []byte) error {
-	newT, err := time.Parse(timeFormat, string(data))
-	if err != nil {
-		return err
-	}
-	*t = Time(newT)
-	return nil
-}
-
-// Error is returned from gofile when things go wrong
-type Error struct {
-	Status string `json:"status"`
-}
-
-// Error returns a string for the error and satisfies the error interface
-func (e Error) Error() string {
-	out := fmt.Sprintf("Error %q", e.Status)
-	return out
-}
-
-// IsError returns true if there is an error
-func (e Error) IsError() bool {
-	return e.Status != "ok"
-}
-
-// Err returns err if not nil, or e if IsError or nil
-func (e Error) Err(err error) error {
-	if err != nil {
-		return err
-	}
-	if e.IsError() {
-		return e
-	}
-	return nil
-}
-
-// Check Error satisfies the error interface
-var _ error = (*Error)(nil)
-
-// Types of things in Item
-const (
-	ItemTypeFolder = "folder"
-	ItemTypeFile   = "file"
-)
-
-// Item describes a folder or a file as returned by /contents
-type Item struct {
-	ID            string                 `json:"id"`
-	ParentFolder  string                 `json:"parentFolder"`
-	Type          string                 `json:"type"`
-	Name          string                 `json:"name"`
-	Size          int64                  `json:"size"`
-	Code          string                 `json:"code"`
-	CreateTime    int64                  `json:"createTime"`
-	ModTime       int64                  `json:"modTime"`
-	Link          string                 `json:"link"`
-	MD5           string                 `json:"md5"`
-	MimeType      string                 `json:"mimetype"`
-	ChildrenCount int                    `json:"childrenCount"`
-	DirectLinks   map[string]*DirectLink `json:"directLinks"`
-	//Public         bool     `json:"public"`
-	//ServerSelected string   `json:"serverSelected"`
-	//Thumbnail      string   `json:"thumbnail"`
-	//DownloadCount int      `json:"downloadCount"`
-	//TotalDownloadCount int64            `json:"totalDownloadCount"`
-	//TotalSize int64            `json:"totalSize"`
-	//ChildrenIDs   []string               `json:"childrenIds"`
-	Children map[string]*Item `json:"children"`
-}
-
-// ToNativeTime converts a go time to a native time
-func ToNativeTime(t time.Time) int64 {
-	return t.Unix()
-}
-
-// FromNativeTime converts native time to a go time
-func FromNativeTime(t int64) time.Time {
-	return time.Unix(t, 0)
-}
-
-// DirectLink describes a direct link to a file so it can be
-// downloaded by third parties.
-type DirectLink struct {
-	ExpireTime       int64  `json:"expireTime"`
-	SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
-	DomainsAllowed   []any  `json:"domainsAllowed"`
-	Auth             []any  `json:"auth"`
-	IsReqLink        bool   `json:"isReqLink"`
-	DirectLink       string `json:"directLink"`
-}
-
-// Contents is returned from the /contents call
-type Contents struct {
-	Error
-	Data struct {
-		Item
-	} `json:"data"`
-	Metadata Metadata `json:"metadata"`
-}
-
-// Metadata is returned when paging is in use
-type Metadata struct {
-	TotalCount  int  `json:"totalCount"`
-	TotalPages  int  `json:"totalPages"`
-	Page        int  `json:"page"`
-	PageSize    int  `json:"pageSize"`
-	HasNextPage bool `json:"hasNextPage"`
-}
-
-// AccountsGetID is the result of /accounts/getid
-type AccountsGetID struct {
-	Error
-	Data struct {
-		ID string `json:"id"`
-	} `json:"data"`
-}
-
-// Stats of storage and traffic
-type Stats struct {
-	FolderCount            int64 `json:"folderCount"`
-	FileCount              int64 `json:"fileCount"`
-	Storage                int64 `json:"storage"`
-	TrafficDirectGenerated int64 `json:"trafficDirectGenerated"`
-	TrafficReqDownloaded   int64 `json:"trafficReqDownloaded"`
-	TrafficWebDownloaded   int64 `json:"trafficWebDownloaded"`
-}
-
-// AccountsGet is the result of /accounts/{id}
-type AccountsGet struct {
-	Error
-	Data struct {
-		ID                             string `json:"id"`
-		Email                          string `json:"email"`
-		Tier                           string `json:"tier"`
-		PremiumType                    string `json:"premiumType"`
-		Token                          string `json:"token"`
-		RootFolder                     string `json:"rootFolder"`
-		SubscriptionProvider           string `json:"subscriptionProvider"`
-		SubscriptionEndDate            int    `json:"subscriptionEndDate"`
-		SubscriptionLimitDirectTraffic int64  `json:"subscriptionLimitDirectTraffic"`
-		SubscriptionLimitStorage       int64  `json:"subscriptionLimitStorage"`
-		StatsCurrent                   Stats  `json:"statsCurrent"`
-		// StatsHistory                   map[int]map[int]map[int]Stats `json:"statsHistory"`
-	} `json:"data"`
-}
-
-// CreateFolderRequest is the input to /contents/createFolder
-type CreateFolderRequest struct {
-	ParentFolderID string `json:"parentFolderId"`
-	FolderName     string `json:"folderName"`
-	ModTime        int64  `json:"modTime,omitempty"`
-}
-
-// CreateFolderResponse is the output from /contents/createFolder
-type CreateFolderResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// DeleteRequest is the input to DELETE /contents
-type DeleteRequest struct {
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// DeleteResponse is the input to DELETE /contents
-type DeleteResponse struct {
-	Error
-	Data map[string]Error
-}
-
-// Server is an upload server
-type Server struct {
-	Name string `json:"name"`
-	Zone string `json:"zone"`
-}
-
-// String returns a string representation of the Server
-func (s *Server) String() string {
-	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
-}
-
-// Root returns the root URL for the server
-func (s *Server) Root() string {
-	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
-}
-
-// URL returns the upload URL for the server
-func (s *Server) URL() string {
-	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
-}
-
-// ServersResponse is the output from /servers
-type ServersResponse struct {
-	Error
-	Data struct {
-		Servers []Server `json:"servers"`
-	} `json:"data"`
-}
-
-// UploadResponse is returned by POST /contents/uploadfile
-type UploadResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// DirectLinksRequest specifies the parameters for the direct link
-type DirectLinksRequest struct {
-	ExpireTime       int64 `json:"expireTime,omitempty"`
-	SourceIpsAllowed []any `json:"sourceIpsAllowed,omitempty"`
-	DomainsAllowed   []any `json:"domainsAllowed,omitempty"`
-	Auth             []any `json:"auth,omitempty"`
-}
-
-// DirectLinksResult is returned from POST /contents/{id}/directlinks
-type DirectLinksResult struct {
-	Error
-	Data struct {
-		ExpireTime       int64  `json:"expireTime"`
-		SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
-		DomainsAllowed   []any  `json:"domainsAllowed"`
-		Auth             []any  `json:"auth"`
-		IsReqLink        bool   `json:"isReqLink"`
-		ID               string `json:"id"`
-		DirectLink       string `json:"directLink"`
-	} `json:"data"`
-}
-
-// UpdateItemRequest describes the updates to be done to an item for PUT /contents/{id}/update
-//
-// The Value of the attribute to define :
-// For Attribute "name" : The name of the content (file or folder)
-// For Attribute "description" : The description displayed on the download page (folder only)
-// For Attribute "tags" : A comma-separated list of tags (folder only)
-// For Attribute "public" : either true or false (folder only)
-// For Attribute "expiry" : A unix timestamp of the expiration date (folder only)
-// For Attribute "password" : The password to set (folder only)
-type UpdateItemRequest struct {
-	Attribute string `json:"attribute"`
-	Value     any    `json:"attributeValue"`
-}
-
-// UpdateItemResponse is returned by PUT /contents/{id}/update
-type UpdateItemResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// MoveRequest is the input to /contents/move
-type MoveRequest struct {
-	FolderID   string `json:"folderId"`
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// MoveResponse is returned by POST /contents/move
-type MoveResponse struct {
-	Error
-	Data map[string]struct {
-		Error
-		Item `json:"data"`
-	} `json:"data"`
-}
-
-// CopyRequest is the input to /contents/copy
-type CopyRequest struct {
-	FolderID   string `json:"folderId"`
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// CopyResponse is returned by POST /contents/copy
-type CopyResponse struct {
-	Error
-	Data map[string]struct {
-		Error
-		Item `json:"data"`
-	} `json:"data"`
-}
-
-// UploadServerStatus is returned when fetching the root of an upload server
-type UploadServerStatus struct {
-	Error
-	Data struct {
-		Server string `json:"server"`
-		Test   string `json:"test"`
-	} `json:"data"`
-}

backend/gofile/gofile_test.go

@@ -1,17 +0,0 @@
-// Test Gofile filesystem interface
-package gofile_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/gofile"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestGoFile:",
-		NilObject:  (*gofile.Object)(nil),
-	})
-}

backend/googlecloudstorage/googlecloudstorage.go

@@ -62,10 +62,9 @@ const (
 
 var (
 	// Description of how to auth for this app
-	storageConfig = &oauthutil.Config{
+	storageConfig = &oauth2.Config{
 		Scopes:       []string{storage.DevstorageReadWriteScope},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
+		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -107,12 +106,6 @@ func init() {
 			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
 			Hide:      fs.OptionHideBoth,
 			Sensitive: true,
-		}, {
-			Name:      "access_token",
-			Help:      "Short-lived access token.\n\nLeave blank normally.\nNeeded only if you want use short-lived access token instead of interactive login.",
-			Hide:      fs.OptionHideConfigurator,
-			Sensitive: true,
-			Advanced:  true,
 		}, {
 			Name:    "anonymous",
 			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",
@@ -386,7 +379,6 @@ type Options struct {
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 	EnvAuth                   bool                 `config:"env_auth"`
 	DirectoryMarkers          bool                 `config:"directory_markers"`
-	AccessToken               string               `config:"access_token"`
 }
 
 // Fs represents a remote storage server
@@ -543,9 +535,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
 		}
-	} else if opt.AccessToken != "" {
-		ts := oauth2.Token{AccessToken: opt.AccessToken}
-		oAuthClient = oauth2.NewClient(ctx, oauth2.StaticTokenSource(&ts))
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {
@@ -708,7 +697,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			// is this a directory marker?
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == f.opt.Enc.ToStandardPath(directory) {
+				if remote == directory {
 					continue
 				}
 				// process directory markers as directories
@@ -955,6 +944,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 		return e
 	}
 	return f.createDirectoryMarker(ctx, bucket, dir)
+
 }
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist

backend/googlephotos/api/types.go

@@ -56,7 +56,8 @@ type MediaItem struct {
 		CreationTime time.Time `json:"creationTime"`
 		Width        string    `json:"width"`
 		Height       string    `json:"height"`
-		Photo        struct{}  `json:"photo"`
+		Photo        struct {
+		} `json:"photo"`
 	} `json:"mediaMetadata"`
 	Filename string `json:"filename"`
 }
@@ -67,7 +68,7 @@ type MediaItems struct {
 	NextPageToken string      `json:"nextPageToken"`
 }
 
-// Content categories
+//Content categories
 // NONE	Default content category. This category is ignored when any other category is used in the filter.
 // LANDSCAPES	Media items containing landscapes.
 // RECEIPTS	Media items containing receipts.
@@ -186,5 +187,5 @@ type BatchCreateResponse struct {
 
 // BatchRemoveItems is for removing items from an album
 type BatchRemoveItems struct {
-	MediaItemIDs []string `json:"mediaItemIds"`
+	MediaItemIds []string `json:"mediaItemIds"`
 }

backend/googlephotos/googlephotos.go

@@ -28,11 +28,13 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 )
 
@@ -59,14 +61,13 @@ const (
 
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauthutil.Config{
+	oauthConfig = &oauth2.Config{
 		Scopes: []string{
 			"openid",
 			"profile",
 			scopeReadWrite, // this must be at position scopeAccess
 		},
-		AuthURL:      google.Endpoint.AuthURL,
-		TokenURL:     google.Endpoint.TokenURL,
+		Endpoint:     google.Endpoint,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
@@ -159,34 +160,6 @@ listings and transferred.
 Without this flag, archived media will not be visible in directory
 listings and won't be transferred.`,
 			Advanced: true,
-		}, {
-			Name:    "proxy",
-			Default: "",
-			Help: strings.ReplaceAll(`Use the gphotosdl proxy for downloading the full resolution images
-
-The Google API will deliver images and video which aren't full
-resolution, and/or have EXIF data missing.
-
-However if you ue the gphotosdl proxy tnen you can download original,
-unchanged images.
-
-This runs a headless browser in the background.
-
-Download the software from [gphotosdl](https://github.com/rclone/gphotosdl)
-
-First run with
-
-    gphotosdl -login
-
-Then once you have logged into google photos close the browser window
-and run
-
-    gphotosdl
-
-Then supply the parameter |--gphotos-proxy "http://localhost:8282"| to make
-rclone use the proxy.
-`, "|", "`"),
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -208,7 +181,6 @@ type Options struct {
 	BatchMode       string               `config:"batch_mode"`
 	BatchSize       int                  `config:"batch_size"`
 	BatchTimeout    fs.Duration          `config:"batch_timeout"`
-	Proxy           string               `config:"proxy"`
 }
 
 // Fs represents a remote storage server
@@ -308,7 +280,7 @@ func errorHandler(resp *http.Response) error {
 	if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
 		body = []byte("Image not found or broken")
 	}
-	e := api.Error{
+	var e = api.Error{
 		Details: api.ErrorDetails{
 			Code:    resp.StatusCode,
 			Message: string(body),
@@ -482,7 +454,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Med
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	// defer log.Trace(f, "remote=%q", remote)("")
+	defer log.Trace(f, "remote=%q", remote)("")
 	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
@@ -648,7 +620,9 @@ func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter
 		if err != nil {
 			return err
 		}
-		entries = append(entries, entry)
+		if entry != nil {
+			entries = append(entries, entry)
+		}
 		return nil
 	})
 	if err != nil {
@@ -695,7 +669,7 @@ func (f *Fs) listUploads(ctx context.Context, dir string) (entries fs.DirEntries
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	// defer log.Trace(f, "dir=%q", dir)("err=%v", &err)
+	defer log.Trace(f, "dir=%q", dir)("err=%v", &err)
 	match, prefix, pattern := patterns.match(f.root, dir, false)
 	if pattern == nil || pattern.isFile {
 		return nil, fs.ErrorDirNotFound
@@ -712,7 +686,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	// defer log.Trace(f, "src=%+v", src)("")
+	defer log.Trace(f, "src=%+v", src)("")
 	// Temporary Object under construction
 	o := &Object{
 		fs:     f,
@@ -728,7 +702,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 		Path:       "/albums",
 		Parameters: url.Values{},
 	}
-	request := api.CreateAlbum{
+	var request = api.CreateAlbum{
 		Album: &api.Album{
 			Title: albumTitle,
 		},
@@ -765,7 +739,7 @@ func (f *Fs) getOrCreateAlbum(ctx context.Context, albumTitle string) (album *ap
 
 // Mkdir creates the album if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
-	// defer log.Trace(f, "dir=%q", dir)("err=%v", &err)
+	defer log.Trace(f, "dir=%q", dir)("err=%v", &err)
 	match, prefix, pattern := patterns.match(f.root, dir, false)
 	if pattern == nil {
 		return fs.ErrorDirNotFound
@@ -789,7 +763,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 //
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
-	// defer log.Trace(f, "dir=%q")("err=%v", &err)
+	defer log.Trace(f, "dir=%q")("err=%v", &err)
 	match, _, pattern := patterns.match(f.root, dir, false)
 	if pattern == nil {
 		return fs.ErrorDirNotFound
@@ -862,7 +836,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	// defer log.Trace(o, "")("")
+	defer log.Trace(o, "")("")
 	if !o.fs.opt.ReadSize || o.bytes >= 0 {
 		return o.bytes
 	}
@@ -963,7 +937,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
-	// defer log.Trace(o, "")("")
+	defer log.Trace(o, "")("")
 	err := o.readMetaData(ctx)
 	if err != nil {
 		fs.Debugf(o, "ModTime: Failed to read metadata: %v", err)
@@ -993,20 +967,16 @@ func (o *Object) downloadURL() string {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	// defer log.Trace(o, "")("")
+	defer log.Trace(o, "")("")
 	err = o.readMetaData(ctx)
 	if err != nil {
 		fs.Debugf(o, "Open: Failed to read metadata: %v", err)
 		return nil, err
 	}
-	url := o.downloadURL()
-	if o.fs.opt.Proxy != "" {
-		url = strings.TrimRight(o.fs.opt.Proxy, "/") + "/id/" + o.id
-	}
 	var resp *http.Response
 	opts := rest.Opts{
 		Method:  "GET",
-		RootURL: url,
+		RootURL: o.downloadURL(),
 		Options: options,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -1032,7 +1002,7 @@ func (f *Fs) commitBatchAlbumID(ctx context.Context, items []uploadedItem, resul
 		Method: "POST",
 		Path:   "/mediaItems:batchCreate",
 	}
-	request := api.BatchCreateRequest{
+	var request = api.BatchCreateRequest{
 		AlbumID: albumID,
 	}
 	itemsInBatch := 0
@@ -1099,7 +1069,7 @@ func (f *Fs) commitBatch(ctx context.Context, items []uploadedItem, results []*a
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	// defer log.Trace(o, "src=%+v", src)("err=%v", &err)
+	defer log.Trace(o, "src=%+v", src)("err=%v", &err)
 	match, _, pattern := patterns.match(o.fs.root, o.remote, true)
 	if pattern == nil || !pattern.isFile || !pattern.canUpload {
 		return errCantUpload
@@ -1168,7 +1138,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		errors := make([]error, 1)
 		results := make([]*api.MediaItem, 1)
 		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
-		if err == nil {
+		if err != nil {
 			err = errors[0]
 			info = results[0]
 		}
@@ -1204,8 +1174,8 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		Path:       "/albums/" + album.ID + ":batchRemoveMediaItems",
 		NoResponse: true,
 	}
-	request := api.BatchRemoveItems{
-		MediaItemIDs: []string{o.id},
+	var request = api.BatchRemoveItems{
+		MediaItemIds: []string{o.id},
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/googlephotos/googlephotos_test.go

@@ -2,7 +2,6 @@ package googlephotos
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,7 +35,7 @@ func TestIntegration(t *testing.T) {
 		*fstest.RemoteName = "TestGooglePhotos:"
 	}
 	f, err := fs.NewFs(ctx, *fstest.RemoteName)
-	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
+	if err == fs.ErrorNotFoundInConfigFile {
 		t.Skipf("Couldn't create google photos backend - skipping tests: %v", err)
 	}
 	require.NoError(t, err)

backend/googlephotos/pattern.go

@@ -38,7 +38,7 @@ type dirPattern struct {
 	toEntries func(ctx context.Context, f lister, prefix string, match []string) (fs.DirEntries, error)
 }
 
-// dirPatterns is a slice of all the directory patterns
+// dirPatters is a slice of all the directory patterns
 type dirPatterns []dirPattern
 
 // patterns describes the layout of the google photos backend file system.

backend/hasher/hasher.go

@@ -164,21 +164,16 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 	}
 
 	stubFeatures := &fs.Features{
-		CanHaveEmptyDirectories:  true,
-		IsLocal:                  true,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CanHaveEmptyDirectories: true,
+		IsLocal:                 true,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -346,22 +341,6 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return errors.New("MergeDirs not supported")
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.Fs.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.Fs.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // DirCacheFlush resets the directory cache - used in testing
 // as an optional interface
 func (f *Fs) DirCacheFlush() {
@@ -439,7 +418,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Shutdown the backend, closing any background tasks and any cached connections.
 func (f *Fs) Shutdown(ctx context.Context) (err error) {
-	if f.db != nil && !f.db.IsStopped() {
+	if f.db != nil {
 		err = f.db.Stop(false)
 	}
 	if do := f.Fs.Features().Shutdown; do != nil {
@@ -535,17 +514,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -562,8 +530,6 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/hasher/object.go

@@ -71,14 +71,7 @@ func (o *Object) Hash(ctx context.Context, hashType hash.Type) (hashVal string,
 	f := o.f
 	if f.passHashes.Contains(hashType) {
 		fs.Debugf(o, "pass %s", hashType)
-		hashVal, err = o.Object.Hash(ctx, hashType)
-		if hashVal != "" {
-			return hashVal, err
-		}
-		if err != nil {
-			fs.Debugf(o, "error passing %s: %v", hashType, err)
-		}
-		fs.Debugf(o, "passed %s is blank -- trying other methods", hashType)
+		return o.Object.Hash(ctx, hashType)
 	}
 	if !f.suppHashes.Contains(hashType) {
 		fs.Debugf(o, "unsupp %s", hashType)

backend/hdfs/fs.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 package hdfs
 
@@ -149,7 +150,7 @@ func (f *Fs) Root() string {
 
 // String returns a description of the FS
 func (f *Fs) String() string {
-	return fmt.Sprintf("hdfs://%s/%s", f.opt.Namenode, f.root)
+	return fmt.Sprintf("hdfs://%s", f.opt.Namenode)
 }
 
 // Features returns the optional features of this Fs
@@ -209,8 +210,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				fs:      f,
 				remote:  remote,
 				size:    x.Size(),
-				modTime: x.ModTime(),
-			})
+				modTime: x.ModTime()})
 		}
 	}
 	return entries, nil

backend/hdfs/hdfs.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 // Package hdfs provides an interface to the HDFS storage system.
 package hdfs

backend/hdfs/hdfs_test.go

@@ -1,6 +1,7 @@
 // Test HDFS filesystem interface
 
 //go:build !plan9
+// +build !plan9
 
 package hdfs_test
 

backend/hdfs/hdfs_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9
+// +build plan9
 
-// Package hdfs provides an interface to the HDFS storage system.
 package hdfs

backend/hdfs/object.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 package hdfs
 

backend/hidrive/hidrive.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -47,9 +48,11 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app.
-	oauthConfig = &oauthutil.Config{
-		AuthURL:      "https://my.hidrive.com/client/authorize",
-		TokenURL:     "https://my.hidrive.com/oauth2/token",
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://my.hidrive.com/client/authorize",
+			TokenURL: "https://my.hidrive.com/oauth2/token",
+		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.TitleBarRedirectURL,
@@ -759,12 +762,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return nil
 }
 
-// Shutdown shutdown the fs
-func (f *Fs) Shutdown(ctx context.Context) error {
-	f.tokenRenewer.Shutdown()
-	return nil
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs.
@@ -1000,7 +997,6 @@ var (
 	_ fs.Copier         = (*Fs)(nil)
 	_ fs.Mover          = (*Fs)(nil)
 	_ fs.DirMover       = (*Fs)(nil)
-	_ fs.Shutdowner     = (*Fs)(nil)
 	_ fs.Object         = (*Object)(nil)
 	_ fs.IDer           = (*Object)(nil)
 )

backend/http/http.go

@@ -89,10 +89,6 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:    "no_escape",
-			Help:    "Do not escape URL metacharacters in path names.",
-			Default: false,
 		}},
 	}
 	fs.Register(fsi)
@@ -104,7 +100,6 @@ type Options struct {
 	NoSlash  bool            `config:"no_slash"`
 	NoHead   bool            `config:"no_head"`
 	Headers  fs.CommaSepList `config:"headers"`
-	NoEscape bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -331,13 +326,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
-	trimmedRemote := strings.TrimLeft(remote, "/") // remove leading "/" since we always have it in f.endpointURL
-	if f.opt.NoEscape {
-		// Directly concatenate without escaping, no_escape behavior
-		return f.endpointURL + trimmedRemote
-	}
-	// Default behavior
-	return f.endpointURL + rest.URLPathEscape(trimmedRemote)
+	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
 // Errors returned by parseName

backend/http/http_internal_test.go

@@ -191,33 +191,6 @@ func TestNewObject(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
-func TestNewObjectWithLeadingSlash(t *testing.T) {
-	f := prepare(t)
-
-	o, err := f.NewObject(context.Background(), "/four/under four.txt")
-	require.NoError(t, err)
-
-	assert.Equal(t, "/four/under four.txt", o.Remote())
-	assert.Equal(t, int64(8+lineEndSize), o.Size())
-	_, ok := o.(*Object)
-	assert.True(t, ok)
-
-	// Test the time is correct on the object
-
-	tObj := o.ModTime(context.Background())
-
-	fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
-	require.NoError(t, err)
-	tFile := fi.ModTime()
-
-	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
-
-	// check object not found
-	o, err = f.NewObject(context.Background(), "/not found.txt")
-	assert.Nil(t, o)
-	assert.Equal(t, fs.ErrorObjectNotFound, err)
-}
-
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

backend/iclouddrive/api/client.go

@@ -1,166 +0,0 @@
-// Package api provides functionality for interacting with the iCloud API.
-package api
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"strings"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	baseEndpoint  = "https://www.icloud.com"
-	homeEndpoint  = "https://www.icloud.com"
-	setupEndpoint = "https://setup.icloud.com/setup/ws/1"
-	authEndpoint  = "https://idmsa.apple.com/appleauth/auth"
-)
-
-type sessionSave func(*Session)
-
-// Client defines the client configuration
-type Client struct {
-	appleID             string
-	password            string
-	srv                 *rest.Client
-	Session             *Session
-	sessionSaveCallback sessionSave
-
-	drive *DriveService
-}
-
-// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback.
-//
-// Parameters:
-// - appleID: the Apple ID of the user.
-// - password: the password of the user.
-// - trustToken: the trust token for the session.
-// - clientID: the client id for the session.
-// - cookies: the cookies for the session.
-// - sessionSaveCallback: the callback function to save the session.
-func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) {
-	icloud := &Client{
-		appleID:             appleID,
-		password:            password,
-		srv:                 rest.NewClient(fshttp.NewClient(context.Background())),
-		Session:             NewSession(),
-		sessionSaveCallback: sessionSaveCallback,
-	}
-
-	icloud.Session.TrustToken = trustToken
-	icloud.Session.Cookies = cookies
-	icloud.Session.ClientID = clientID
-	return icloud, nil
-}
-
-// DriveService returns the DriveService instance associated with the Client.
-func (c *Client) DriveService() (*DriveService, error) {
-	var err error
-	if c.drive == nil {
-		c.drive, err = NewDriveService(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return c.drive, nil
-}
-
-// Request makes a request and retries it if the session is invalid.
-//
-// This function is the main entry point for making requests to the iCloud
-// API. If the initial request returns a 401 (Unauthorized), it will try to
-// reauthenticate and retry the request.
-func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
-	resp, err = c.Session.Request(ctx, opts, request, response)
-	if err != nil && resp != nil {
-		// try to reauth
-		if resp.StatusCode == 401 || resp.StatusCode == 421 {
-			err = c.Authenticate(ctx)
-			if err != nil {
-				return nil, err
-			}
-
-			if c.Session.Requires2FA() {
-				return nil, errors.New("trust token expired, please reauth")
-			}
-			return c.RequestNoReAuth(ctx, opts, request, response)
-		}
-	}
-	return resp, err
-}
-
-// RequestNoReAuth makes a request without re-authenticating.
-//
-// This function is useful when you have a session that is already
-// authenticated, but you need to make a request without triggering
-// a re-authentication.
-func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
-	// Make the request without re-authenticating
-	resp, err = c.Session.Request(ctx, opts, request, response)
-	return resp, err
-}
-
-// Authenticate authenticates the client with the iCloud API.
-func (c *Client) Authenticate(ctx context.Context) error {
-	if c.Session.Cookies != nil {
-		if err := c.Session.ValidateSession(ctx); err == nil {
-			fs.Debugf("icloud", "Valid session, no need to reauth")
-			return nil
-		}
-		c.Session.Cookies = nil
-	}
-
-	fs.Debugf("icloud", "Authenticating as %s\n", c.appleID)
-	err := c.Session.SignIn(ctx, c.appleID, c.password)
-
-	if err == nil {
-		err = c.Session.AuthWithToken(ctx)
-		if err == nil && c.sessionSaveCallback != nil {
-			c.sessionSaveCallback(c.Session)
-		}
-	}
-	return err
-}
-
-// SignIn signs in the client using the provided context and credentials.
-func (c *Client) SignIn(ctx context.Context) error {
-	return c.Session.SignIn(ctx, c.appleID, c.password)
-}
-
-// IntoReader marshals the provided values into a JSON encoded reader
-func IntoReader(values any) (*bytes.Reader, error) {
-	m, err := json.Marshal(values)
-	if err != nil {
-		return nil, err
-	}
-	return bytes.NewReader(m), nil
-}
-
-// RequestError holds info on a result state, icloud can return a 200 but the result is unknown
-type RequestError struct {
-	Status string
-	Text   string
-}
-
-// Error satisfy the error interface.
-func (e *RequestError) Error() string {
-	return fmt.Sprintf("%s: %s", e.Text, e.Status)
-}
-
-func newRequestError(Status string, Text string) *RequestError {
-	return &RequestError{
-		Status: strings.ToLower(Status),
-		Text:   Text,
-	}
-}
-
-// newErr orf makes a new error from sprintf parameters.
-func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError {
-	return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
-}

backend/iclouddrive/api/drive.go

@@ -1,913 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"mime"
-	"net/http"
-	"net/url"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	defaultZone        = "com.apple.CloudDocs"
-	statusOk           = "OK"
-	statusEtagConflict = "ETAG_CONFLICT"
-)
-
-// DriveService represents an iCloud Drive service.
-type DriveService struct {
-	icloud       *Client
-	RootID       string
-	endpoint     string
-	docsEndpoint string
-}
-
-// NewDriveService creates a new DriveService instance.
-func NewDriveService(icloud *Client) (*DriveService, error) {
-	return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices["drivews"].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices["docws"].URL}, nil
-}
-
-// GetItemByDriveID retrieves a DriveItem by its Drive ID.
-func (d *DriveService) GetItemByDriveID(ctx context.Context, id string, includeChildren bool) (*DriveItem, *http.Response, error) {
-	items, resp, err := d.GetItemsByDriveID(ctx, []string{id}, includeChildren)
-	if err != nil {
-		return nil, resp, err
-	}
-	return items[0], resp, err
-}
-
-// GetItemsByDriveID retrieves DriveItems by their Drive IDs.
-func (d *DriveService) GetItemsByDriveID(ctx context.Context, ids []string, includeChildren bool) ([]*DriveItem, *http.Response, error) {
-	var err error
-	_items := []map[string]any{}
-	for _, id := range ids {
-		_items = append(_items, map[string]any{
-			"drivewsid":        id,
-			"partialData":      false,
-			"includeHierarchy": false,
-		})
-	}
-
-	var body *bytes.Reader
-	var path string
-	if !includeChildren {
-		values := []map[string]any{{
-			"items": _items,
-		}}
-		body, err = IntoReader(values)
-		if err != nil {
-			return nil, nil, err
-		}
-		path = "/retrieveItemDetails"
-	} else {
-		values := _items
-		body, err = IntoReader(values)
-		if err != nil {
-			return nil, nil, err
-		}
-		path = "/retrieveItemDetailsInFolders"
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         path,
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.endpoint,
-		Body:         body,
-	}
-	var items []*DriveItem
-	resp, err := d.icloud.Request(ctx, opts, nil, &items)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return items, resp, err
-}
-
-// GetDocByPath retrieves a document by its path.
-func (d *DriveService) GetDocByPath(ctx context.Context, path string) (*Document, *http.Response, error) {
-	values := url.Values{}
-	values.Set("unified_format", "false")
-	body, err := IntoReader(path)
-	if err != nil {
-		return nil, nil, err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/list/lookup_by_path",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Parameters:   values,
-		Body:         body,
-	}
-	var item []*Document
-	resp, err := d.icloud.Request(ctx, opts, nil, &item)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return item[0], resp, err
-}
-
-// GetItemByPath retrieves a DriveItem by its path.
-func (d *DriveService) GetItemByPath(ctx context.Context, path string) (*DriveItem, *http.Response, error) {
-	values := url.Values{}
-	values.Set("unified_format", "true")
-
-	body, err := IntoReader(path)
-	if err != nil {
-		return nil, nil, err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/list/lookup_by_path",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Parameters:   values,
-		Body:         body,
-	}
-	var item []*DriveItem
-	resp, err := d.icloud.Request(ctx, opts, nil, &item)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return item[0], resp, err
-}
-
-// GetDocByItemID retrieves a document by its item ID.
-func (d *DriveService) GetDocByItemID(ctx context.Context, id string) (*Document, *http.Response, error) {
-	values := url.Values{}
-	values.Set("document_id", id)
-	values.Set("unified_format", "false") // important
-	opts := rest.Opts{
-		Method:       "GET",
-		Path:         "/ws/" + defaultZone + "/list/lookup_by_id",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Parameters:   values,
-	}
-	var item *Document
-	resp, err := d.icloud.Request(ctx, opts, nil, &item)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return item, resp, err
-}
-
-// GetItemRawByItemID retrieves a DriveItemRaw by its item ID.
-func (d *DriveService) GetItemRawByItemID(ctx context.Context, id string) (*DriveItemRaw, *http.Response, error) {
-	opts := rest.Opts{
-		Method:       "GET",
-		Path:         "/v1/item/" + id,
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-	}
-	var item *DriveItemRaw
-	resp, err := d.icloud.Request(ctx, opts, nil, &item)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return item, resp, err
-}
-
-// GetItemsInFolder retrieves a list of DriveItemRaw objects in a folder with the given ID.
-func (d *DriveService) GetItemsInFolder(ctx context.Context, id string, limit int64) ([]*DriveItemRaw, *http.Response, error) {
-	values := url.Values{}
-	values.Set("limit", strconv.FormatInt(limit, 10))
-
-	opts := rest.Opts{
-		Method:       "GET",
-		Path:         "/v1/enumerate/" + id,
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Parameters:   values,
-	}
-
-	items := struct {
-		Items []*DriveItemRaw `json:"drive_item"`
-	}{}
-
-	resp, err := d.icloud.Request(ctx, opts, nil, &items)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return items.Items, resp, err
-}
-
-// GetDownloadURLByDriveID retrieves the download URL for a file in the DriveService.
-func (d *DriveService) GetDownloadURLByDriveID(ctx context.Context, id string) (string, *http.Response, error) {
-	_, zone, docid := DeconstructDriveID(id)
-	values := url.Values{}
-	values.Set("document_id", docid)
-
-	if zone == "" {
-		zone = defaultZone
-	}
-
-	opts := rest.Opts{
-		Method:       "GET",
-		Path:         "/ws/" + zone + "/download/by_id",
-		Parameters:   values,
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-	}
-
-	var filer *FileRequest
-	resp, err := d.icloud.Request(ctx, opts, nil, &filer)
-
-	if err != nil {
-		return "", resp, err
-	}
-
-	var url string
-	if filer.DataToken != nil {
-		url = filer.DataToken.URL
-	} else {
-		url = filer.PackageToken.URL
-	}
-
-	return url, resp, err
-}
-
-// DownloadFile downloads a file from the given URL using the provided options.
-func (d *DriveService) DownloadFile(ctx context.Context, url string, opt []fs.OpenOption) (*http.Response, error) {
-	opts := &rest.Opts{
-		Method:       "GET",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      url,
-		Options:      opt,
-	}
-
-	resp, err := d.icloud.srv.Call(ctx, opts)
-	if err != nil {
-		// icloud has some weird http codes
-		if resp.StatusCode == 330 {
-			loc, err := resp.Location()
-			if err == nil {
-				return d.DownloadFile(ctx, loc.String(), opt)
-			}
-		}
-
-		return resp, err
-	}
-	return d.icloud.srv.Call(ctx, opts)
-}
-
-// MoveItemToTrashByItemID moves an item to the trash based on the item ID.
-func (d *DriveService) MoveItemToTrashByItemID(ctx context.Context, id, etag string, force bool) (*DriveItem, *http.Response, error) {
-	doc, resp, err := d.GetDocByItemID(ctx, id)
-	if err != nil {
-		return nil, resp, err
-	}
-	return d.MoveItemToTrashByID(ctx, doc.DriveID(), etag, force)
-}
-
-// MoveItemToTrashByID moves an item to the trash based on the item ID.
-func (d *DriveService) MoveItemToTrashByID(ctx context.Context, drivewsid, etag string, force bool) (*DriveItem, *http.Response, error) {
-	values := map[string]any{
-		"items": []map[string]any{{
-			"drivewsid": drivewsid,
-			"etag":      etag,
-			"clientId":  drivewsid,
-		}}}
-
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/moveItemsToTrash",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.endpoint,
-		Body:         body,
-	}
-
-	item := struct {
-		Items []*DriveItem `json:"items"`
-	}{}
-	resp, err := d.icloud.Request(ctx, opts, nil, &item)
-
-	if err != nil {
-		return nil, resp, err
-	}
-
-	if item.Items[0].Status != statusOk {
-		// rerun with latest etag
-		if force && item.Items[0].Status == "ETAG_CONFLICT" {
-			return d.MoveItemToTrashByID(ctx, drivewsid, item.Items[0].Etag, false)
-		}
-
-		err = newRequestError(item.Items[0].Status, "unknown request status")
-	}
-
-	return item.Items[0], resp, err
-}
-
-// CreateNewFolderByItemID creates a new folder by item ID.
-func (d *DriveService) CreateNewFolderByItemID(ctx context.Context, id, name string) (*DriveItem, *http.Response, error) {
-	doc, resp, err := d.GetDocByItemID(ctx, id)
-	if err != nil {
-		return nil, resp, err
-	}
-	return d.CreateNewFolderByDriveID(ctx, doc.DriveID(), name)
-}
-
-// CreateNewFolderByDriveID creates a new folder by its Drive ID.
-func (d *DriveService) CreateNewFolderByDriveID(ctx context.Context, drivewsid, name string) (*DriveItem, *http.Response, error) {
-	values := map[string]any{
-		"destinationDrivewsId": drivewsid,
-		"folders": []map[string]any{{
-			"clientId": "FOLDER::UNKNOWN_ZONE::TempId-" + uuid.New().String(),
-			"name":     name,
-		}},
-	}
-
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/createFolders",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.endpoint,
-		Body:         body,
-	}
-	var fResp *CreateFoldersResponse
-	resp, err := d.icloud.Request(ctx, opts, nil, &fResp)
-	if err != nil {
-		return nil, resp, err
-	}
-	status := fResp.Folders[0].Status
-	if status != statusOk {
-		err = newRequestError(status, "unknown request status")
-	}
-
-	return fResp.Folders[0], resp, err
-}
-
-// RenameItemByItemID renames a DriveItem by its item ID.
-func (d *DriveService) RenameItemByItemID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) {
-	doc, resp, err := d.GetDocByItemID(ctx, id)
-	if err != nil {
-		return nil, resp, err
-	}
-	return d.RenameItemByDriveID(ctx, doc.DriveID(), doc.Etag, name, force)
-}
-
-// RenameItemByDriveID renames a DriveItem by its drive ID.
-func (d *DriveService) RenameItemByDriveID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) {
-	values := map[string]any{
-		"items": []map[string]any{{
-			"drivewsid": id,
-			"name":      name,
-			"etag":      etag,
-			// "extension": split[1],
-		}},
-	}
-
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/renameItems",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.endpoint,
-		Body:         body,
-	}
-	var items *DriveItem
-	resp, err := d.icloud.Request(ctx, opts, nil, &items)
-
-	if err != nil {
-		return nil, resp, err
-	}
-
-	status := items.Items[0].Status
-	if status != statusOk {
-		// rerun with latest etag
-		if force && status == "ETAG_CONFLICT" {
-			return d.RenameItemByDriveID(ctx, id, items.Items[0].Etag, name, false)
-		}
-		err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL)
-	}
-
-	return items.Items[0], resp, err
-}
-
-// MoveItemByItemID moves an item by its item ID to a destination item ID.
-func (d *DriveService) MoveItemByItemID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) {
-	docSrc, resp, err := d.GetDocByItemID(ctx, id)
-	if err != nil {
-		return nil, resp, err
-	}
-	docDst, resp, err := d.GetDocByItemID(ctx, dstID)
-	if err != nil {
-		return nil, resp, err
-	}
-	return d.MoveItemByDriveID(ctx, docSrc.DriveID(), docSrc.Etag, docDst.DriveID(), force)
-}
-
-// MoveItemByDocID moves an item by its doc ID.
-// func (d *DriveService) MoveItemByDocID(ctx context.Context, srcDocID, srcEtag, dstDocID string, force bool) (*DriveItem, *http.Response, error) {
-// 	return d.MoveItemByDriveID(ctx, srcDocID, srcEtag, docDst.DriveID(), force)
-// }
-
-// MoveItemByDriveID moves an item by its drive ID.
-func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) {
-	values := map[string]any{
-		"destinationDrivewsId": dstID,
-		"items": []map[string]any{{
-			"drivewsid": id,
-			"etag":      etag,
-			"clientId":  id,
-		}},
-	}
-
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/moveItems",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.endpoint,
-		Body:         body,
-	}
-
-	var items *DriveItem
-	resp, err := d.icloud.Request(ctx, opts, nil, &items)
-
-	if err != nil {
-		return nil, resp, err
-	}
-
-	status := items.Items[0].Status
-	if status != statusOk {
-		// rerun with latest etag
-		if force && status == "ETAG_CONFLICT" {
-			return d.MoveItemByDriveID(ctx, id, items.Items[0].Etag, dstID, false)
-		}
-		err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL)
-	}
-
-	return items.Items[0], resp, err
-}
-
-// CopyDocByItemID copies a document by its item ID.
-func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) {
-	// putting name in info doesnt work. extension does work so assume this is a bug in the endpoint
-	values := map[string]any{
-		"info_to_update": map[string]any{},
-	}
-
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/v1/item/copy/" + itemID,
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Body:         body,
-	}
-
-	var info *DriveItemRaw
-	resp, err := d.icloud.Request(ctx, opts, nil, &info)
-	if err != nil {
-		return nil, resp, err
-	}
-	return info, resp, err
-}
-
-// CreateUpload creates an url for an upload.
-func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) {
-	// first we need to request an upload url
-	values := map[string]any{
-		"filename":     name,
-		"type":         "FILE",
-		"size":         strconv.FormatInt(size, 10),
-		"content_type": GetContentTypeForFile(name),
-	}
-	body, err := IntoReader(values)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/upload/web",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Body:         body,
-	}
-	var responseInfo []*UploadResponse
-	resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo)
-	if err != nil {
-		return nil, resp, err
-	}
-	return responseInfo[0], resp, err
-}
-
-// Upload uploads a file to the given url
-func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, name, uploadURL string) (*SingleFileResponse, *http.Response, error) {
-	// TODO: implement multipart upload
-	opts := rest.Opts{
-		Method:        "POST",
-		ExtraHeaders:  d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:       uploadURL,
-		Body:          in,
-		ContentLength: &size,
-		ContentType:   GetContentTypeForFile(name),
-		// MultipartContentName: "files",
-		MultipartFileName: name,
-	}
-	var singleFileResponse *SingleFileResponse
-	resp, err := d.icloud.Request(ctx, opts, nil, &singleFileResponse)
-	if err != nil {
-		return nil, resp, err
-	}
-	return singleFileResponse, resp, err
-}
-
-// UpdateFile updates a file in the DriveService.
-//
-// ctx: the context.Context object for the request.
-// r: a pointer to the UpdateFileInfo struct containing the information for the file update.
-// Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any.
-func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) {
-	body, err := IntoReader(r)
-	if err != nil {
-		return nil, nil, err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/update/documents",
-		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
-		RootURL:      d.docsEndpoint,
-		Body:         body,
-	}
-	var responseInfo *DocumentUpdateResponse
-	resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	doc := responseInfo.Results[0].Document
-	item := DriveItem{
-		Drivewsid:    "FILE::com.apple.CloudDocs::" + doc.DocumentID,
-		Docwsid:      doc.DocumentID,
-		Itemid:       doc.ItemID,
-		Etag:         doc.Etag,
-		ParentID:     doc.ParentID,
-		DateModified: time.Unix(r.Mtime, 0),
-		DateCreated:  time.Unix(r.Mtime, 0),
-		Type:         doc.Type,
-		Name:         doc.Name,
-		Size:         doc.Size,
-	}
-
-	return &item, resp, err
-}
-
-// UpdateFileInfo represents the information for an update to a file in the DriveService.
-type UpdateFileInfo struct {
-	AllowConflict   bool   `json:"allow_conflict"`
-	Btime           int64  `json:"btime"`
-	Command         string `json:"command"`
-	CreateShortGUID bool   `json:"create_short_guid"`
-	Data            struct {
-		Receipt            string `json:"receipt,omitempty"`
-		ReferenceSignature string `json:"reference_signature,omitempty"`
-		Signature          string `json:"signature,omitempty"`
-		Size               int64  `json:"size,omitempty"`
-		WrappingKey        string `json:"wrapping_key,omitempty"`
-	} `json:"data,omitempty"`
-	DocumentID string    `json:"document_id"`
-	FileFlags  FileFlags `json:"file_flags"`
-	Mtime      int64     `json:"mtime"`
-	Path       struct {
-		Path               string `json:"path"`
-		StartingDocumentID string `json:"starting_document_id"`
-	} `json:"path"`
-}
-
-// FileFlags defines the file flags for a document.
-type FileFlags struct {
-	IsExecutable bool `json:"is_executable"`
-	IsHidden     bool `json:"is_hidden"`
-	IsWritable   bool `json:"is_writable"`
-}
-
-// NewUpdateFileInfo creates a new UpdateFileInfo object with default values.
-//
-// Returns an UpdateFileInfo object.
-func NewUpdateFileInfo() UpdateFileInfo {
-	return UpdateFileInfo{
-		Command:         "add_file",
-		CreateShortGUID: true,
-		AllowConflict:   true,
-		FileFlags: FileFlags{
-			IsExecutable: true,
-			IsHidden:     false,
-			IsWritable:   false,
-		},
-	}
-}
-
-// DriveItemRaw is a raw drive item.
-// not suure what to call this but there seems to be a "unified" and non "unified" drive item response. This is the non unified.
-type DriveItemRaw struct {
-	ItemID   string            `json:"item_id"`
-	ItemInfo *DriveItemRawInfo `json:"item_info"`
-}
-
-// SplitName splits the name of a DriveItemRaw into its name and extension.
-//
-// It returns the name and extension as separate strings. If the name ends with a dot,
-// it means there is no extension, so an empty string is returned for the extension.
-// If the name does not contain a dot, it means
-func (d *DriveItemRaw) SplitName() (string, string) {
-	name := d.ItemInfo.Name
-	// ends with a dot, no extension
-	if strings.HasSuffix(name, ".") {
-		return name, ""
-	}
-	lastInd := strings.LastIndex(name, ".")
-
-	if lastInd == -1 {
-		return name, ""
-	}
-	return name[:lastInd], name[lastInd+1:]
-}
-
-// ModTime returns the modification time of the DriveItemRaw.
-//
-// It parses the ModifiedAt field of the ItemInfo struct and converts it to a time.Time value.
-// If the parsing fails, it returns the zero value of time.Time.
-// The returned time.Time value represents the modification time of the DriveItemRaw.
-func (d *DriveItemRaw) ModTime() time.Time {
-	i, err := strconv.ParseInt(d.ItemInfo.ModifiedAt, 10, 64)
-	if err != nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(i)
-}
-
-// CreatedTime returns the creation time of the DriveItemRaw.
-//
-// It parses the CreatedAt field of the ItemInfo struct and converts it to a time.Time value.
-// If the parsing fails, it returns the zero value of time.Time.
-// The returned time.Time
-func (d *DriveItemRaw) CreatedTime() time.Time {
-	i, err := strconv.ParseInt(d.ItemInfo.CreatedAt, 10, 64)
-	if err != nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(i)
-}
-
-// DriveItemRawInfo is the raw information about a drive item.
-type DriveItemRawInfo struct {
-	Name string `json:"name"`
-	// Extension is absolutely borked on endpoints so dont use it.
-	Extension  string `json:"extension"`
-	Size       int64  `json:"size,string"`
-	Type       string `json:"type"`
-	Version    string `json:"version"`
-	ModifiedAt string `json:"modified_at"`
-	CreatedAt  string `json:"created_at"`
-	Urls       struct {
-		URLDownload string `json:"url_download"`
-	} `json:"urls"`
-}
-
-// IntoDriveItem converts a DriveItemRaw into a DriveItem.
-//
-// It takes no parameters.
-// It returns a pointer to a DriveItem.
-func (d *DriveItemRaw) IntoDriveItem() *DriveItem {
-	name, extension := d.SplitName()
-	return &DriveItem{
-		Itemid:       d.ItemID,
-		Name:         name,
-		Extension:    extension,
-		Type:         d.ItemInfo.Type,
-		Etag:         d.ItemInfo.Version,
-		DateModified: d.ModTime(),
-		DateCreated:  d.CreatedTime(),
-		Size:         d.ItemInfo.Size,
-		Urls:         d.ItemInfo.Urls,
-	}
-}
-
-// DocumentUpdateResponse is the response of a document update request.
-type DocumentUpdateResponse struct {
-	Status struct {
-		StatusCode   int    `json:"status_code"`
-		ErrorMessage string `json:"error_message"`
-	} `json:"status"`
-	Results []struct {
-		Status struct {
-			StatusCode   int    `json:"status_code"`
-			ErrorMessage string `json:"error_message"`
-		} `json:"status"`
-		OperationID interface{} `json:"operation_id"`
-		Document    *Document   `json:"document"`
-	} `json:"results"`
-}
-
-// Document represents a document on iCloud.
-type Document struct {
-	Status struct {
-		StatusCode   int    `json:"status_code"`
-		ErrorMessage string `json:"error_message"`
-	} `json:"status"`
-	DocumentID string `json:"document_id"`
-	ItemID     string `json:"item_id"`
-	Urls       struct {
-		URLDownload string `json:"url_download"`
-	} `json:"urls"`
-	Etag           string       `json:"etag"`
-	ParentID       string       `json:"parent_id"`
-	Name           string       `json:"name"`
-	Type           string       `json:"type"`
-	Deleted        bool         `json:"deleted"`
-	Mtime          int64        `json:"mtime"`
-	LastEditorName string       `json:"last_editor_name"`
-	Data           DocumentData `json:"data"`
-	Size           int64        `json:"size"`
-	Btime          int64        `json:"btime"`
-	Zone           string       `json:"zone"`
-	FileFlags      struct {
-		IsExecutable bool `json:"is_executable"`
-		IsWritable   bool `json:"is_writable"`
-		IsHidden     bool `json:"is_hidden"`
-	} `json:"file_flags"`
-	LastOpenedTime   int64       `json:"lastOpenedTime"`
-	RestorePath      interface{} `json:"restorePath"`
-	HasChainedParent bool        `json:"hasChainedParent"`
-}
-
-// DriveID returns the drive ID of the Document.
-func (d *Document) DriveID() string {
-	if d.Zone == "" {
-		d.Zone = defaultZone
-	}
-	return d.Type + "::" + d.Zone + "::" + d.DocumentID
-}
-
-// DocumentData represents the data of a document.
-type DocumentData struct {
-	Signature          string `json:"signature"`
-	Owner              string `json:"owner"`
-	Size               int64  `json:"size"`
-	ReferenceSignature string `json:"reference_signature"`
-	WrappingKey        string `json:"wrapping_key"`
-	PcsInfo            string `json:"pcsInfo"`
-}
-
-// SingleFileResponse is the response of a single file request.
-type SingleFileResponse struct {
-	SingleFile *SingleFileInfo `json:"singleFile"`
-}
-
-// SingleFileInfo represents the information of a single file.
-type SingleFileInfo struct {
-	ReferenceSignature string `json:"referenceChecksum"`
-	Size               int64  `json:"size"`
-	Signature          string `json:"fileChecksum"`
-	WrappingKey        string `json:"wrappingKey"`
-	Receipt            string `json:"receipt"`
-}
-
-// UploadResponse is the response of an upload request.
-type UploadResponse struct {
-	URL        string `json:"url"`
-	DocumentID string `json:"document_id"`
-}
-
-// FileRequestToken represents the token of a file request.
-type FileRequestToken struct {
-	URL                string `json:"url"`
-	Token              string `json:"token"`
-	Signature          string `json:"signature"`
-	WrappingKey        string `json:"wrapping_key"`
-	ReferenceSignature string `json:"reference_signature"`
-}
-
-// FileRequest represents the request of a file.
-type FileRequest struct {
-	DocumentID   string            `json:"document_id"`
-	ItemID       string            `json:"item_id"`
-	OwnerDsid    int64             `json:"owner_dsid"`
-	DataToken    *FileRequestToken `json:"data_token,omitempty"`
-	PackageToken *FileRequestToken `json:"package_token,omitempty"`
-	DoubleEtag   string            `json:"double_etag"`
-}
-
-// CreateFoldersResponse is the response of a create folders request.
-type CreateFoldersResponse struct {
-	Folders []*DriveItem `json:"folders"`
-}
-
-// DriveItem represents an item on iCloud.
-type DriveItem struct {
-	DateCreated         time.Time    `json:"dateCreated"`
-	Drivewsid           string       `json:"drivewsid"`
-	Docwsid             string       `json:"docwsid"`
-	Itemid              string       `json:"item_id"`
-	Zone                string       `json:"zone"`
-	Name                string       `json:"name"`
-	ParentID            string       `json:"parentId"`
-	Hierarchy           []DriveItem  `json:"hierarchy"`
-	Etag                string       `json:"etag"`
-	Type                string       `json:"type"`
-	AssetQuota          int64        `json:"assetQuota"`
-	FileCount           int64        `json:"fileCount"`
-	ShareCount          int64        `json:"shareCount"`
-	ShareAliasCount     int64        `json:"shareAliasCount"`
-	DirectChildrenCount int64        `json:"directChildrenCount"`
-	Items               []*DriveItem `json:"items"`
-	NumberOfItems       int64        `json:"numberOfItems"`
-	Status              string       `json:"status"`
-	Extension           string       `json:"extension,omitempty"`
-	DateModified        time.Time    `json:"dateModified,omitempty"`
-	DateChanged         time.Time    `json:"dateChanged,omitempty"`
-	Size                int64        `json:"size,omitempty"`
-	LastOpenTime        time.Time    `json:"lastOpenTime,omitempty"`
-	Urls                struct {
-		URLDownload string `json:"url_download"`
-	} `json:"urls"`
-}
-
-// IsFolder returns true if the item is a folder.
-func (d *DriveItem) IsFolder() bool {
-	return d.Type == "FOLDER" || d.Type == "APP_CONTAINER" || d.Type == "APP_LIBRARY"
-}
-
-// DownloadURL returns the download URL of the item.
-func (d *DriveItem) DownloadURL() string {
-	return d.Urls.URLDownload
-}
-
-// FullName returns the full name of the item.
-// name + extension
-func (d *DriveItem) FullName() string {
-	if d.Extension != "" {
-		return d.Name + "." + d.Extension
-	}
-	return d.Name
-}
-
-// GetDocIDFromDriveID returns the DocumentID from the drive ID.
-func GetDocIDFromDriveID(id string) string {
-	split := strings.Split(id, "::")
-	return split[len(split)-1]
-}
-
-// DeconstructDriveID returns the document type, zone, and document ID from the drive ID.
-func DeconstructDriveID(id string) (docType, zone, docid string) {
-	split := strings.Split(id, "::")
-	if len(split) < 3 {
-		return "", "", id
-	}
-	return split[0], split[1], split[2]
-}
-
-// ConstructDriveID constructs a drive ID from the given components.
-func ConstructDriveID(id string, zone string, t string) string {
-	return strings.Join([]string{t, zone, id}, "::")
-}
-
-// GetContentTypeForFile detects content type for given file name.
-func GetContentTypeForFile(name string) string {
-	// detect MIME type by looking at the filename only
-	mimeType := mime.TypeByExtension(filepath.Ext(name))
-	if mimeType == "" {
-		// api requires a mime type passed in
-		mimeType = "text/plain"
-	}
-	return strings.Split(mimeType, ";")[0]
-}

backend/iclouddrive/api/session.go

@@ -1,412 +0,0 @@
-package api
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"slices"
-	"strings"
-
-	"github.com/oracle/oci-go-sdk/v65/common"
-
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// Session represents an iCloud session
-type Session struct {
-	SessionToken   string         `json:"session_token"`
-	Scnt           string         `json:"scnt"`
-	SessionID      string         `json:"session_id"`
-	AccountCountry string         `json:"account_country"`
-	TrustToken     string         `json:"trust_token"`
-	ClientID       string         `json:"client_id"`
-	Cookies        []*http.Cookie `json:"cookies"`
-	AccountInfo    AccountInfo    `json:"account_info"`
-
-	srv *rest.Client `json:"-"`
-}
-
-// String returns the session as a string
-// func (s *Session) String() string {
-// 	jsession, _ := json.Marshal(s)
-// 	return string(jsession)
-// }
-
-// Request makes a request
-func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) {
-	resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
-
-	if err != nil {
-		return resp, err
-	}
-
-	if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" {
-		s.AccountCountry = val
-	}
-	if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" {
-		s.SessionID = val
-	}
-	if val := resp.Header.Get("X-Apple-Session-Token"); val != "" {
-		s.SessionToken = val
-	}
-	if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" {
-		s.TrustToken = val
-	}
-	if val := resp.Header.Get("scnt"); val != "" {
-		s.Scnt = val
-	}
-
-	return resp, nil
-}
-
-// Requires2FA returns true if the session requires 2FA
-func (s *Session) Requires2FA() bool {
-	return s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired
-}
-
-// SignIn signs in the session
-func (s *Session) SignIn(ctx context.Context, appleID, password string) error {
-	trustTokens := []string{}
-	if s.TrustToken != "" {
-		trustTokens = []string{s.TrustToken}
-	}
-	values := map[string]any{
-		"accountName": appleID,
-		"password":    password,
-		"rememberMe":  true,
-		"trustTokens": trustTokens,
-	}
-	body, err := IntoReader(values)
-	if err != nil {
-		return err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/signin",
-		Parameters:   url.Values{},
-		ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
-		RootURL:      authEndpoint,
-		IgnoreStatus: true, // need to handle 409 for hsa2
-		NoResponse:   true,
-		Body:         body,
-	}
-	opts.Parameters.Set("isRememberMeEnabled", "true")
-	_, err = s.Request(ctx, opts, nil, nil)
-
-	return err
-
-}
-
-// AuthWithToken authenticates the session
-func (s *Session) AuthWithToken(ctx context.Context) error {
-	values := map[string]any{
-		"accountCountryCode": s.AccountCountry,
-		"dsWebAuthToken":     s.SessionToken,
-		"extended_login":     true,
-		"trustToken":         s.TrustToken,
-	}
-	body, err := IntoReader(values)
-	if err != nil {
-		return err
-	}
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/accountLogin",
-		ExtraHeaders: GetCommonHeaders(map[string]string{}),
-		RootURL:      setupEndpoint,
-		Body:         body,
-	}
-
-	resp, err := s.Request(ctx, opts, nil, &s.AccountInfo)
-	if err == nil {
-		s.Cookies = resp.Cookies()
-	}
-
-	return err
-}
-
-// Validate2FACode validates the 2FA code
-func (s *Session) Validate2FACode(ctx context.Context, code string) error {
-	values := map[string]interface{}{"securityCode": map[string]string{"code": code}}
-	body, err := IntoReader(values)
-	if err != nil {
-		return err
-	}
-
-	headers := s.GetAuthHeaders(map[string]string{})
-	headers["scnt"] = s.Scnt
-	headers["X-Apple-ID-Session-Id"] = s.SessionID
-
-	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "/verify/trusteddevice/securitycode",
-		ExtraHeaders: headers,
-		RootURL:      authEndpoint,
-		Body:         body,
-		NoResponse:   true,
-	}
-
-	_, err = s.Request(ctx, opts, nil, nil)
-	if err == nil {
-		if err := s.TrustSession(ctx); err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	return fmt.Errorf("validate2FACode failed: %w", err)
-}
-
-// TrustSession trusts the session
-func (s *Session) TrustSession(ctx context.Context) error {
-	headers := s.GetAuthHeaders(map[string]string{})
-	headers["scnt"] = s.Scnt
-	headers["X-Apple-ID-Session-Id"] = s.SessionID
-
-	opts := rest.Opts{
-		Method:        "GET",
-		Path:          "/2sv/trust",
-		ExtraHeaders:  headers,
-		RootURL:       authEndpoint,
-		NoResponse:    true,
-		ContentLength: common.Int64(0),
-	}
-
-	_, err := s.Request(ctx, opts, nil, nil)
-	if err != nil {
-		return fmt.Errorf("trustSession failed: %w", err)
-	}
-
-	return s.AuthWithToken(ctx)
-}
-
-// ValidateSession validates the session
-func (s *Session) ValidateSession(ctx context.Context) error {
-	opts := rest.Opts{
-		Method:        "POST",
-		Path:          "/validate",
-		ExtraHeaders:  s.GetHeaders(map[string]string{}),
-		RootURL:       setupEndpoint,
-		ContentLength: common.Int64(0),
-	}
-	_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
-	if err != nil {
-		return fmt.Errorf("validateSession failed: %w", err)
-	}
-
-	return nil
-}
-
-// GetAuthHeaders returns the authentication headers for the session.
-//
-// It takes an `overwrite` map[string]string parameter which allows
-// overwriting the default headers. It returns a map[string]string.
-func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string {
-	headers := map[string]string{
-		"Accept":                           "application/json",
-		"Content-Type":                     "application/json",
-		"X-Apple-OAuth-Client-Id":          s.ClientID,
-		"X-Apple-OAuth-Client-Type":        "firstPartyAuth",
-		"X-Apple-OAuth-Redirect-URI":       "https://www.icloud.com",
-		"X-Apple-OAuth-Require-Grant-Code": "true",
-		"X-Apple-OAuth-Response-Mode":      "web_message",
-		"X-Apple-OAuth-Response-Type":      "code",
-		"X-Apple-OAuth-State":              s.ClientID,
-		"X-Apple-Widget-Key":               s.ClientID,
-		"Origin":                           homeEndpoint,
-		"Referer":                          fmt.Sprintf("%s/", homeEndpoint),
-		"User-Agent":                       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
-	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
-	return headers
-}
-
-// GetHeaders Gets the authentication headers required for a request
-func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
-	headers := GetCommonHeaders(map[string]string{})
-	headers["Cookie"] = s.GetCookieString()
-	for k, v := range overwrite {
-		headers[k] = v
-	}
-	return headers
-}
-
-// GetCookieString returns the cookie header string for the session.
-func (s *Session) GetCookieString() string {
-	cookieHeader := ""
-	// we only care about name and value.
-	for _, cookie := range s.Cookies {
-		cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";"
-	}
-	return cookieHeader
-}
-
-// GetCommonHeaders generates common HTTP headers with optional overwrite.
-func GetCommonHeaders(overwrite map[string]string) map[string]string {
-	headers := map[string]string{
-		"Content-Type": "application/json",
-		"Origin":       baseEndpoint,
-		"Referer":      fmt.Sprintf("%s/", baseEndpoint),
-		"User-Agent":   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
-	}
-	for k, v := range overwrite {
-		headers[k] = v
-	}
-	return headers
-}
-
-// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added.
-func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) {
-	var hashes []string
-	for _, cookie := range right {
-		hashes = append(hashes, cookie.Raw)
-	}
-	for _, cookie := range left {
-		if !slices.Contains(hashes, cookie.Raw) {
-			right = append(right, cookie)
-		}
-	}
-	return right, nil
-}
-
-// GetCookiesForDomain filters the provided cookies based on the domain of the given URL.
-func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) {
-	var domainCookies []*http.Cookie
-	for _, cookie := range cookies {
-		if strings.HasSuffix(url.Host, cookie.Domain) {
-			domainCookies = append(domainCookies, cookie)
-		}
-	}
-	return domainCookies, nil
-}
-
-// NewSession creates a new Session instance with default values.
-func NewSession() *Session {
-	session := &Session{}
-	session.srv = rest.NewClient(fshttp.NewClient(context.Background())).SetRoot(baseEndpoint)
-	//session.ClientID = "auth-" + uuid.New().String()
-	return session
-}
-
-// AccountInfo represents an account info
-type AccountInfo struct {
-	DsInfo                       *ValidateDataDsInfo    `json:"dsInfo"`
-	HasMinimumDeviceForPhotosWeb bool                   `json:"hasMinimumDeviceForPhotosWeb"`
-	ICDPEnabled                  bool                   `json:"iCDPEnabled"`
-	Webservices                  map[string]*webService `json:"webservices"`
-	PcsEnabled                   bool                   `json:"pcsEnabled"`
-	TermsUpdateNeeded            bool                   `json:"termsUpdateNeeded"`
-	ConfigBag                    struct {
-		Urls struct {
-			AccountCreateUI     string `json:"accountCreateUI"`
-			AccountLoginUI      string `json:"accountLoginUI"`
-			AccountLogin        string `json:"accountLogin"`
-			AccountRepairUI     string `json:"accountRepairUI"`
-			DownloadICloudTerms string `json:"downloadICloudTerms"`
-			RepairDone          string `json:"repairDone"`
-			AccountAuthorizeUI  string `json:"accountAuthorizeUI"`
-			VettingURLForEmail  string `json:"vettingUrlForEmail"`
-			AccountCreate       string `json:"accountCreate"`
-			GetICloudTerms      string `json:"getICloudTerms"`
-			VettingURLForPhone  string `json:"vettingUrlForPhone"`
-		} `json:"urls"`
-		AccountCreateEnabled bool `json:"accountCreateEnabled"`
-	} `json:"configBag"`
-	HsaTrustedBrowser            bool     `json:"hsaTrustedBrowser"`
-	AppsOrder                    []string `json:"appsOrder"`
-	Version                      int      `json:"version"`
-	IsExtendedLogin              bool     `json:"isExtendedLogin"`
-	PcsServiceIdentitiesIncluded bool     `json:"pcsServiceIdentitiesIncluded"`
-	IsRepairNeeded               bool     `json:"isRepairNeeded"`
-	HsaChallengeRequired         bool     `json:"hsaChallengeRequired"`
-	RequestInfo                  struct {
-		Country  string `json:"country"`
-		TimeZone string `json:"timeZone"`
-		Region   string `json:"region"`
-	} `json:"requestInfo"`
-	PcsDeleted bool `json:"pcsDeleted"`
-	ICloudInfo struct {
-		SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"`
-	} `json:"iCloudInfo"`
-	Apps map[string]*ValidateDataApp `json:"apps"`
-}
-
-// ValidateDataDsInfo represents an validation info
-type ValidateDataDsInfo struct {
-	HsaVersion                         int           `json:"hsaVersion"`
-	LastName                           string        `json:"lastName"`
-	ICDPEnabled                        bool          `json:"iCDPEnabled"`
-	TantorMigrated                     bool          `json:"tantorMigrated"`
-	Dsid                               string        `json:"dsid"`
-	HsaEnabled                         bool          `json:"hsaEnabled"`
-	IsHideMyEmailSubscriptionActive    bool          `json:"isHideMyEmailSubscriptionActive"`
-	IroncadeMigrated                   bool          `json:"ironcadeMigrated"`
-	Locale                             string        `json:"locale"`
-	BrZoneConsolidated                 bool          `json:"brZoneConsolidated"`
-	ICDRSCapableDeviceList             string        `json:"ICDRSCapableDeviceList"`
-	IsManagedAppleID                   bool          `json:"isManagedAppleID"`
-	IsCustomDomainsFeatureAvailable    bool          `json:"isCustomDomainsFeatureAvailable"`
-	IsHideMyEmailFeatureAvailable      bool          `json:"isHideMyEmailFeatureAvailable"`
-	ContinueOnDeviceEligibleDeviceInfo []string      `json:"ContinueOnDeviceEligibleDeviceInfo"`
-	Gilligvited                        bool          `json:"gilligvited"`
-	AppleIDAliases                     []interface{} `json:"appleIdAliases"`
-	UbiquityEOLEnabled                 bool          `json:"ubiquityEOLEnabled"`
-	IsPaidDeveloper                    bool          `json:"isPaidDeveloper"`
-	CountryCode                        string        `json:"countryCode"`
-	NotificationID                     string        `json:"notificationId"`
-	PrimaryEmailVerified               bool          `json:"primaryEmailVerified"`
-	ADsID                              string        `json:"aDsID"`
-	Locked                             bool          `json:"locked"`
-	ICDRSCapableDeviceCount            int           `json:"ICDRSCapableDeviceCount"`
-	HasICloudQualifyingDevice          bool          `json:"hasICloudQualifyingDevice"`
-	PrimaryEmail                       string        `json:"primaryEmail"`
-	AppleIDEntries                     []struct {
-		IsPrimary bool   `json:"isPrimary"`
-		Type      string `json:"type"`
-		Value     string `json:"value"`
-	} `json:"appleIdEntries"`
-	GilliganEnabled    bool   `json:"gilligan-enabled"`
-	IsWebAccessAllowed bool   `json:"isWebAccessAllowed"`
-	FullName           string `json:"fullName"`
-	MailFlags          struct {
-		IsThreadingAvailable           bool `json:"isThreadingAvailable"`
-		IsSearchV2Provisioned          bool `json:"isSearchV2Provisioned"`
-		SCKMail                        bool `json:"sCKMail"`
-		IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"`
-	} `json:"mailFlags"`
-	LanguageCode         string `json:"languageCode"`
-	AppleID              string `json:"appleId"`
-	HasUnreleasedOS      bool   `json:"hasUnreleasedOS"`
-	AnalyticsOptInStatus bool   `json:"analyticsOptInStatus"`
-	FirstName            string `json:"firstName"`
-	ICloudAppleIDAlias   string `json:"iCloudAppleIdAlias"`
-	NotesMigrated        bool   `json:"notesMigrated"`
-	BeneficiaryInfo      struct {
-		IsBeneficiary bool `json:"isBeneficiary"`
-	} `json:"beneficiaryInfo"`
-	HasPaymentInfo bool   `json:"hasPaymentInfo"`
-	PcsDelet       bool   `json:"pcsDelet"`
-	AppleIDAlias   string `json:"appleIdAlias"`
-	BrMigrated     bool   `json:"brMigrated"`
-	StatusCode     int    `json:"statusCode"`
-	FamilyEligible bool   `json:"familyEligible"`
-}
-
-// ValidateDataApp represents an app
-type ValidateDataApp struct {
-	CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"`
-	IsQualifiedForBeta     bool `json:"isQualifiedForBeta"`
-}
-
-// WebService represents a web service
-type webService struct {
-	PcsRequired bool   `json:"pcsRequired"`
-	URL         string `json:"url"`
-	UploadURL   string `json:"uploadUrl"`
-	Status      string `json:"status"`
-}

backend/iclouddrive/iclouddrive_test.go

@@ -1,18 +0,0 @@
-//go:build !plan9 && !solaris
-
-package iclouddrive_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/iclouddrive"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestICloudDrive:",
-		NilObject:  (*iclouddrive.Object)(nil),
-	})
-}

backend/iclouddrive/iclouddrive_unsupported.go

@@ -1,7 +0,0 @@
-// Build for iclouddrive for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-//go:build plan9 || solaris
-
-// Package iclouddrive implements the iCloud Drive backend
-package iclouddrive

backend/imagekit/client/media.go

@@ -75,7 +75,7 @@ type MoveFolderParam struct {
 	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
 }
 
-// JobIDResponse represents response struct with JobID for folder operations
+// JobIDResponse respresents response struct with JobID for folder operations
 type JobIDResponse struct {
 	JobID string `json:"jobId"`
 }

backend/imagekit/client/url.go

@@ -56,7 +56,7 @@ func (ik *ImageKit) URL(params URLParam) (string, error) {
 		var expires = strconv.FormatInt(now+params.ExpireSeconds, 10)
 		var path = strings.Replace(resultURL, endpoint, "", 1)
 
-		path += expires
+		path = path + expires
 		mac := hmac.New(sha1.New, []byte(ik.PrivateKey))
 		mac.Write([]byte(path))
 		signature := hex.EncodeToString(mac.Sum(nil))

backend/jottacloud/jottacloud.go

@@ -277,9 +277,11 @@ machines.`)
 		m.Set(configClientID, teliaseCloudClientID)
 		m.Set(configTokenURL, teliaseCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     teliaseCloudAuthURL,
-				TokenURL:    teliaseCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  teliaseCloudAuthURL,
+					TokenURL: teliaseCloudTokenURL,
+				},
 				ClientID:    teliaseCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -290,9 +292,11 @@ machines.`)
 		m.Set(configClientID, telianoCloudClientID)
 		m.Set(configTokenURL, telianoCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     telianoCloudAuthURL,
-				TokenURL:    telianoCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  telianoCloudAuthURL,
+					TokenURL: telianoCloudTokenURL,
+				},
 				ClientID:    telianoCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -303,9 +307,11 @@ machines.`)
 		m.Set(configClientID, tele2CloudClientID)
 		m.Set(configTokenURL, tele2CloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     tele2CloudAuthURL,
-				TokenURL:    tele2CloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  tele2CloudAuthURL,
+					TokenURL: tele2CloudTokenURL,
+				},
 				ClientID:    tele2CloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -316,9 +322,11 @@ machines.`)
 		m.Set(configClientID, onlimeCloudClientID)
 		m.Set(configTokenURL, onlimeCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauthutil.Config{
-				AuthURL:     onlimeCloudAuthURL,
-				TokenURL:    onlimeCloudTokenURL,
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  onlimeCloudAuthURL,
+					TokenURL: onlimeCloudTokenURL,
+				},
 				ClientID:    onlimeCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -916,17 +924,19 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	}
 
 	baseClient := fshttp.NewClient(ctx)
-	oauthConfig := &oauthutil.Config{
-		AuthURL:  defaultTokenURL,
-		TokenURL: defaultTokenURL,
+	oauthConfig := &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
+		},
 	}
 	if ver == configVersion {
 		oauthConfig.ClientID = defaultClientID
 		// if custom endpoints are set use them else stick with defaults
 		if tokenURL, ok := m.Get(configTokenURL); ok {
-			oauthConfig.TokenURL = tokenURL
+			oauthConfig.Endpoint.TokenURL = tokenURL
 			// jottacloud is weird. we need to use the tokenURL as authURL
-			oauthConfig.AuthURL = tokenURL
+			oauthConfig.Endpoint.AuthURL = tokenURL
 		}
 	} else if ver == legacyConfigVersion {
 		clientID, ok := m.Get(configClientID)
@@ -940,8 +950,8 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 		oauthConfig.ClientID = clientID
 		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-		oauthConfig.TokenURL = legacyTokenURL
-		oauthConfig.AuthURL = legacyTokenURL
+		oauthConfig.Endpoint.TokenURL = legacyTokenURL
+		oauthConfig.Endpoint.AuthURL = legacyTokenURL
 
 		// add the request filter to fix token refresh
 		if do, ok := baseClient.Transport.(interface {
@@ -1477,38 +1487,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	err := f.mkParentDir(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
-
-	if err := f.mkParentDir(ctx, remote); err != nil {
-		return nil, err
-	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
-	if err == nil {
-		var createTime time.Time
-		var createTimeMeta bool
-		var modTime time.Time
-		var modTimeMeta bool
-		if meta != nil {
-			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
-			if !createTimeMeta {
-				createTime = srcObj.createTime
-			}
-			modTime, modTimeMeta = srcObj.parseFsMetadataTime(meta, "mtime")
-			if !modTimeMeta {
-				modTime = srcObj.modTime
-			}
-		}
-		if bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
-			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
-			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		} else if createTimeMeta || modTimeMeta {
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		}
+	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
+	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
+		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
+		info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
 	}
 
 	if err != nil {
@@ -1535,30 +1523,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	err := f.mkParentDir(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
-
-	if err := f.mkParentDir(ctx, remote); err != nil {
-		return nil, err
-	}
 	info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)
 
-	if err == nil && meta != nil {
-		createTime, createTimeMeta := srcObj.parseFsMetadataTime(meta, "btime")
-		if !createTimeMeta {
-			createTime = srcObj.createTime
-		}
-		modTime, modTimeMeta := srcObj.parseFsMetadataTime(meta, "mtime")
-		if !modTimeMeta {
-			modTime = srcObj.modTime
-		}
-		if createTimeMeta || modTimeMeta {
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		}
-	}
-
 	if err != nil {
 		return nil, fmt.Errorf("couldn't move file: %w", err)
 	}
@@ -1710,12 +1680,6 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return nil
 }
 
-// Shutdown shutdown the fs
-func (f *Fs) Shutdown(ctx context.Context) error {
-	f.tokenRenewer.Shutdown()
-	return nil
-}
-
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
@@ -1816,20 +1780,6 @@ func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
 	return o.setMetaData(info)
 }
 
-// parseFsMetadataTime parses a time string from fs.Metadata with key
-func (o *Object) parseFsMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
-	value, ok := m[key]
-	if ok {
-		var err error
-		t, err = time.Parse(time.RFC3339Nano, value) // metadata stores RFC3339Nano timestamps
-		if err != nil {
-			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
-			ok = false
-		}
-	}
-	return t, ok
-}
-
 // ModTime returns the modification time of the object
 //
 // It attempts to read the objects mtime and if that isn't present the
@@ -2001,11 +1951,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var createdTime string
 	var modTime string
 	if meta != nil {
-		if t, ok := o.parseFsMetadataTime(meta, "btime"); ok {
-			createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+		if v, ok := meta["btime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
+			} else {
+				createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+			}
 		}
-		if t, ok := o.parseFsMetadataTime(meta, "mtime"); ok {
-			modTime = api.Rfc3339Time(t).String()
+		if v, ok := meta["mtime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v)
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
+			} else {
+				modTime = api.Rfc3339Time(t).String()
+			}
 		}
 	}
 	if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
@@ -2144,7 +2104,6 @@ var (
 	_ fs.Abouter      = (*Fs)(nil)
 	_ fs.UserInfoer   = (*Fs)(nil)
 	_ fs.CleanUpper   = (*Fs)(nil)
-	_ fs.Shutdowner   = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
 	_ fs.MimeTyper    = (*Object)(nil)
 	_ fs.Metadataer   = (*Object)(nil)

backend/jottacloud/jottacloud_internal_test.go

@@ -59,7 +59,7 @@ func (f *Fs) InternalTestMetadata(t *testing.T) {
 		//"utime" - read-only
 		//"content-type" - read-only
 	}
-	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, false, contents, true, "text/html", metadata)
+	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata)
 	defer func() {
 		assert.NoError(t, obj.Remove(ctx))
 	}()

backend/koofr/koofr.go

@@ -67,13 +67,13 @@ func init() {
 			Sensitive: true,
 		}, {
 			Name:       "password",
-			Help:       "Your password for rclone generate one at https://app.koofr.net/app/admin/preferences/password.",
+			Help:       "Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password).",
 			Provider:   "koofr",
 			IsPassword: true,
 			Required:   true,
 		}, {
 			Name:       "password",
-			Help:       "Your password for rclone generate one at https://storage.rcs-rds.ro/app/admin/preferences/password.",
+			Help:       "Your password for rclone (generate one at https://storage.rcs-rds.ro/app/admin/preferences/password).",
 			Provider:   "digistorage",
 			IsPassword: true,
 			Required:   true,

backend/linkbox/linkbox.go

@@ -36,7 +36,7 @@ import (
 )
 
 const (
-	maxEntitiesPerPage = 1000
+	maxEntitiesPerPage = 1024
 	minSleep           = 200 * time.Millisecond
 	maxSleep           = 2 * time.Second
 	pacerBurst         = 1
@@ -219,8 +219,7 @@ type listAllFn func(*entity) bool
 // Search is a bit fussy about which characters match
 //
 // If the name doesn't match this then do an dir list instead
-// N.B.: Linkbox doesn't support search by name that is longer than 50 chars
-var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ -.]{1,50}$`)
+var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ .]+$`)
 
 // Lists the directory required calling the user function on each item found
 //
@@ -239,7 +238,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllF
 		// If name isn't good then do an unbounded search
 		name = ""
 	}
-
 OUTER:
 	for numberOfEntities == maxEntitiesPerPage {
 		pageNumber++
@@ -260,6 +258,7 @@ OUTER:
 		err = getUnmarshaledResponse(ctx, f, opts, &responseResult)
 		if err != nil {
 			return false, fmt.Errorf("getting files failed: %w", err)
+
 		}
 
 		numberOfEntities = len(responseResult.SearchData.Entities)

backend/linkbox/linkbox_test.go

@@ -13,7 +13,5 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestLinkbox:",
 		NilObject:  (*linkbox.Object)(nil),
-		// Linkbox doesn't support leading dots for files
-		SkipLeadingDot: true,
 	})
 }