lib/rest: retry errors due to failed JSON decoding - fixes #8383

These errors are almost always caused by an overloaded proxy somewhere
in the chain returning HTML so they are worth retrying.

.github/workflows/build_publish_docker_plugin.yml

@@ -7,43 +7,39 @@ name: Release Build for Docker Plugin
 on:
   release:
     types: [published]
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        default: true
 
 jobs:
-  build_docker_volume_plugin:
-    if: inputs.manual || github.repository == 'rclone/rclone'
-    name: Build docker plugin job
-    runs-on: ubuntu-latest
-    steps:
-      - name: Free some space
-        shell: bash
-        run: |
-          df -h .
-          # Remove android SDK
-          sudo rm -rf /usr/local/lib/android || true
-          # Remove .net runtime
-          sudo rm -rf /usr/share/dotnet || true
-          df -h .
-      - name: Checkout master
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Build and publish docker plugin
-        shell: bash
-        run: |
-          VER=${GITHUB_REF#refs/tags/}
-          PLUGIN_USER=rclone
-          docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                       --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-          for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-              export PLUGIN_USER PLUGIN_ARCH
-              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-          done
-          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}
+
+    build_docker_volume_plugin:
+        if: github.repository == 'rclone/rclone'
+        needs: build
+        runs-on: ubuntu-latest
+        name: Build docker plugin job
+        steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
+            - name: Checkout master
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Build and publish docker plugin
+              shell: bash
+              run: |
+                VER=${GITHUB_REF#refs/tags/}
+                PLUGIN_USER=rclone
+                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+                    export PLUGIN_USER PLUGIN_ARCH
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+                done
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

RELEASE.md

@@ -131,8 +131,8 @@ Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * make startstable
   * Do the steps as above
+  * make startstable
   * git co master
   * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/ftp"
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"

docs/content/authors.md

@@ -944,5 +944,3 @@ put them back in again.` >}}
   * Joel K Biju <joelkbiju18@gmail.com>
   * ll3006 <doublel3006@gmail.com>
   * jbagwell-akamai <113531113+jbagwell-akamai@users.noreply.github.com>
-  * Michael Kebe <michael.kebe@gmail.com>
-  * Lorenz Brun <lorenz@brun.one>

docs/content/changelog.md

@@ -5,32 +5,6 @@ description: "Rclone Changelog"
 
 # Changelog
 
-## v1.69.1 - 2025-02-14
-
-[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1)
-
-* Bug Fixes
-    * lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood)
-    * bisync: Fix listings missing concurrent modifications (nielash)
-    * serve s3: Fix list objects encoding-type (Nick Craig-Wood)
-    * fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood)
-    * doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies)
-    * build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal)
-* VFS
-    * Fix the cache failing to upload symlinks when `--links` was specified (Nick Craig-Wood)
-    * Fix race detected by race detector (Nick Craig-Wood)
-    * Close the change notify channel on Shutdown (izouxv)
-* B2
-    * Fix "fatal error: concurrent map writes" (Nick Craig-Wood)
-* Iclouddrive
-    * Add notes on ADP and Missing PCS cookies (Nick Craig-Wood)
-* Onedrive
-    * Mark German (de) region as deprecated (Nick Craig-Wood)
-* S3
-    * Added new storage class to magalu provider (Bruno Fernandes)
-    * Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe)
-    * Add latest Linode Object Storage endpoints (jbagwell-akamai)
-
 ## v1.69.0 - 2025-01-12
 
 [See commits](https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0)

docs/content/sftp.md

@@ -1108,8 +1108,7 @@ Properties:
 
 On some SFTP servers (e.g. Synology) the paths are different
 for SSH and SFTP so the hashes can't be calculated properly.
-You can either use [`--sftp-path-override`](#--sftp-path-override)
-or [`disable_hashcheck`](#--sftp-disable-hashcheck).
+For them using `disable_hashcheck` is a good idea.
 
 The only ssh agent supported under Windows is Putty's pageant.
 

fs/accounting/stats_groups_test.go

@@ -237,14 +237,5 @@ func TestCountError(t *testing.T) {
 }
 
 func percentDiff(start, end uint64) uint64 {
-	if start == 0 {
-		return 0 // Handle zero start value to avoid division by zero
-	}
-	var diff uint64
-	if end > start {
-		diff = end - start // Handle case where end is larger than start
-	} else {
-		diff = start - end
-	}
-	return (diff * 100) / start
+	return (start - end) * 100 / start
 }

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/henrybear327/Proton-API-Bridge v1.0.0
 	github.com/henrybear327/go-proton-api v1.0.0
 	github.com/jcmturner/gokrb5/v8 v8.4.4
+	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3
 	github.com/josephspurrier/goversioninfo v1.4.1
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
 	github.com/klauspost/compress v1.17.11
@@ -59,7 +60,6 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
-	github.com/rclone/ftp v0.0.0-20250219192741-48e008274523
 	github.com/rclone/gofakes3 v0.0.4
 	github.com/rfjakob/eme v1.1.2
 	github.com/rivo/uniseg v0.4.7

go.sum

@@ -406,6 +406,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
+github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
+github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
 github.com/josephspurrier/goversioninfo v1.4.1 h1:5LvrkP+n0tg91J9yTkoVnt/QgNnrI1t4uSsWjIonrqY=
 github.com/josephspurrier/goversioninfo v1.4.1/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -533,10 +535,6 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1
 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
-github.com/rclone/ftp v0.0.0-20250219192741-48e008274523 h1:RUQMVDOZzJdyE+jBtiPz2aOSA1fmhKci2D04yDDc5VA=
-github.com/rclone/ftp v0.0.0-20250219192741-48e008274523/go.mod h1:tkIuFVxqqZ+cjUkGfCPvOwEz6nmN1B6Lm6KnB9YU8nQ=
-github.com/rclone/ftp v1.0.0-220913-160 h1:DLkuGd3+5lvC1LccUQM35465OH2Hitlm35eI9q/VS2M=
-github.com/rclone/ftp v1.0.0-220913-160/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8=
 github.com/rclone/gofakes3 v0.0.4 h1:LswpC49VY/UJ1zucoL5ktnOEX6lq3qK7e1aFIAfqCbk=
 github.com/rclone/gofakes3 v0.0.4/go.mod h1:j/UoS+2/Mr7xAlfKhyVC58YyFQmh9uoQA5YZQXQUqmg=
 github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=

lib/rest/rest.go

@@ -17,6 +17,7 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -189,7 +190,13 @@ func checkDrainAndClose(r io.ReadCloser, err *error) {
 func DecodeJSON(resp *http.Response, result interface{}) (err error) {
 	defer checkDrainAndClose(resp.Body, &err)
 	decoder := json.NewDecoder(resp.Body)
-	return decoder.Decode(result)
+	err = decoder.Decode(result)
+	if err != nil {
+		// Retry this as it is likely some overloaded server sending HTML instead of JSON
+		contentType := resp.Header.Get("Content-Type")
+		err = fserrors.RetryError(fmt.Errorf("failed to decode JSON from Content-Type: %q: %v", contentType, err))
+	}
+	return err
 }
 
 // DecodeXML decodes resp.Body into result

vfs/dir.go

@@ -760,7 +760,6 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 			dir := node.(*Dir)
 			dir.mu.Lock()
 			dir.modTime = item.ModTime(context.TODO())
-			dir.entry = item
 			if dirTree != nil {
 				err = dir._readDirFromDirTree(dirTree, when)
 				if err != nil {

vfs/dir_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"runtime"
 	"sort"
 	"testing"
 	"time"
@@ -656,34 +655,3 @@ func TestDirFileOpen(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int64(12), fi.Size())
 }
-
-func TestDirEntryModTimeInvalidation(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("dirent modtime is unreliable on Windows filesystems")
-	}
-	r, vfs := newTestVFS(t)
-
-	// Needs to be less than 2x the wait time below, othewrwise the entry
-	// gets cleared out before it had a chance to be updated.
-	vfs.Opt.DirCacheTime = fs.Duration(50 * time.Millisecond)
-
-	r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
-
-	node, err := vfs.Stat("dir")
-	require.NoError(t, err)
-	modTime1 := node.(*Dir).DirEntry().ModTime(context.Background())
-
-	// Wait some time, then write another file which must update ModTime of
-	// the directory.
-	time.Sleep(75 * time.Millisecond)
-	r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2)
-
-	node2, err := vfs.Stat("dir")
-	require.NoError(t, err)
-	modTime2 := node2.(*Dir).DirEntry().ModTime(context.Background())
-
-	// ModTime of directory must be different after second file was written.
-	if modTime1.Equal(modTime2) {
-		t.Error("ModTime not invalidated")
-	}
-}