ftp: fix listings on servers where MLST has multiple spaces - fixes #8403 FIXME NEEDS UPSTREAM PATCH BEFORE MERGE

The VFS directory cache layer didn't update directory entry properties
if they are reused after cache invalidation.

Update them unconditionally as newDir sets them to the same value and
setting a pointer is cheaper in both LoC as well as CPU cycles than a
branch.

Also add a test exercising this behavior.

Fixes #6335

.github/workflows/build_publish_docker_plugin.yml

@@ -7,39 +7,43 @@ name: Release Build for Docker Plugin
 on:
   release:
     types: [published]
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
 
 jobs:
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}
+  build_docker_volume_plugin:
+    if: inputs.manual || github.repository == 'rclone/rclone'
+    name: Build docker plugin job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free some space
+        shell: bash
+        run: |
+          df -h .
+          # Remove android SDK
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove .net runtime
+          sudo rm -rf /usr/share/dotnet || true
+          df -h .
+      - name: Checkout master
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build and publish docker plugin
+        shell: bash
+        run: |
+          VER=${GITHUB_REF#refs/tags/}
+          PLUGIN_USER=rclone
+          docker login --username ${{ secrets.DOCKER_HUB_USER }} \
+                       --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
+          for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
+              export PLUGIN_USER PLUGIN_ARCH
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
+              make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+          done
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
+          make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

RELEASE.md

@@ -131,8 +131,8 @@ Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * Do the steps as above
   * make startstable
+  * Do the steps as above
   * git co master
   * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/jlaffaye/ftp"
+	"github.com/rclone/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"

docs/content/authors.md

@@ -944,3 +944,5 @@ put them back in again.` >}}
   * Joel K Biju <joelkbiju18@gmail.com>
   * ll3006 <doublel3006@gmail.com>
   * jbagwell-akamai <113531113+jbagwell-akamai@users.noreply.github.com>
+  * Michael Kebe <michael.kebe@gmail.com>
+  * Lorenz Brun <lorenz@brun.one>

docs/content/changelog.md

@@ -5,6 +5,32 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.69.1 - 2025-02-14
+
+[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1)
+
+* Bug Fixes
+    * lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood)
+    * bisync: Fix listings missing concurrent modifications (nielash)
+    * serve s3: Fix list objects encoding-type (Nick Craig-Wood)
+    * fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood)
+    * doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies)
+    * build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal)
+* VFS
+    * Fix the cache failing to upload symlinks when `--links` was specified (Nick Craig-Wood)
+    * Fix race detected by race detector (Nick Craig-Wood)
+    * Close the change notify channel on Shutdown (izouxv)
+* B2
+    * Fix "fatal error: concurrent map writes" (Nick Craig-Wood)
+* Iclouddrive
+    * Add notes on ADP and Missing PCS cookies (Nick Craig-Wood)
+* Onedrive
+    * Mark German (de) region as deprecated (Nick Craig-Wood)
+* S3
+    * Added new storage class to magalu provider (Bruno Fernandes)
+    * Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe)
+    * Add latest Linode Object Storage endpoints (jbagwell-akamai)
+
 ## v1.69.0 - 2025-01-12
 
 [See commits](https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0)

docs/content/sftp.md

@@ -1108,7 +1108,8 @@ Properties:
 
 On some SFTP servers (e.g. Synology) the paths are different
 for SSH and SFTP so the hashes can't be calculated properly.
-For them using `disable_hashcheck` is a good idea.
+You can either use [`--sftp-path-override`](#--sftp-path-override)
+or [`disable_hashcheck`](#--sftp-disable-hashcheck).
 
 The only ssh agent supported under Windows is Putty's pageant.
 

fs/accounting/stats_groups_test.go

@@ -237,5 +237,14 @@ func TestCountError(t *testing.T) {
 }
 
 func percentDiff(start, end uint64) uint64 {
-	return (start - end) * 100 / start
+	if start == 0 {
+		return 0 // Handle zero start value to avoid division by zero
+	}
+	var diff uint64
+	if end > start {
+		diff = end - start // Handle case where end is larger than start
+	} else {
+		diff = start - end
+	}
+	return (diff * 100) / start
 }

go.mod

@@ -41,7 +41,6 @@ require (
 	github.com/henrybear327/Proton-API-Bridge v1.0.0
 	github.com/henrybear327/go-proton-api v1.0.0
 	github.com/jcmturner/gokrb5/v8 v8.4.4
-	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3
 	github.com/josephspurrier/goversioninfo v1.4.1
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
 	github.com/klauspost/compress v1.17.11
@@ -60,6 +59,7 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
+	github.com/rclone/ftp v0.0.0-20250219192741-48e008274523
 	github.com/rclone/gofakes3 v0.0.4
 	github.com/rfjakob/eme v1.1.2
 	github.com/rivo/uniseg v0.4.7

go.sum

@@ -406,8 +406,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
-github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
-github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
 github.com/josephspurrier/goversioninfo v1.4.1 h1:5LvrkP+n0tg91J9yTkoVnt/QgNnrI1t4uSsWjIonrqY=
 github.com/josephspurrier/goversioninfo v1.4.1/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -535,6 +533,10 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1
 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
+github.com/rclone/ftp v0.0.0-20250219192741-48e008274523 h1:RUQMVDOZzJdyE+jBtiPz2aOSA1fmhKci2D04yDDc5VA=
+github.com/rclone/ftp v0.0.0-20250219192741-48e008274523/go.mod h1:tkIuFVxqqZ+cjUkGfCPvOwEz6nmN1B6Lm6KnB9YU8nQ=
+github.com/rclone/ftp v1.0.0-220913-160 h1:DLkuGd3+5lvC1LccUQM35465OH2Hitlm35eI9q/VS2M=
+github.com/rclone/ftp v1.0.0-220913-160/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8=
 github.com/rclone/gofakes3 v0.0.4 h1:LswpC49VY/UJ1zucoL5ktnOEX6lq3qK7e1aFIAfqCbk=
 github.com/rclone/gofakes3 v0.0.4/go.mod h1:j/UoS+2/Mr7xAlfKhyVC58YyFQmh9uoQA5YZQXQUqmg=
 github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=

lib/rest/rest.go

@@ -17,7 +17,6 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -190,13 +189,7 @@ func checkDrainAndClose(r io.ReadCloser, err *error) {
 func DecodeJSON(resp *http.Response, result interface{}) (err error) {
 	defer checkDrainAndClose(resp.Body, &err)
 	decoder := json.NewDecoder(resp.Body)
-	err = decoder.Decode(result)
-	if err != nil {
-		// Retry this as it is likely some overloaded server sending HTML instead of JSON
-		contentType := resp.Header.Get("Content-Type")
-		err = fserrors.RetryError(fmt.Errorf("failed to decode JSON from Content-Type: %q: %v", contentType, err))
-	}
-	return err
+	return decoder.Decode(result)
 }
 
 // DecodeXML decodes resp.Body into result

vfs/dir.go

@@ -760,6 +760,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 			dir := node.(*Dir)
 			dir.mu.Lock()
 			dir.modTime = item.ModTime(context.TODO())
+			dir.entry = item
 			if dirTree != nil {
 				err = dir._readDirFromDirTree(dirTree, when)
 				if err != nil {

vfs/dir_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 	"sort"
 	"testing"
 	"time"
@@ -655,3 +656,34 @@ func TestDirFileOpen(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int64(12), fi.Size())
 }
+
+func TestDirEntryModTimeInvalidation(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("dirent modtime is unreliable on Windows filesystems")
+	}
+	r, vfs := newTestVFS(t)
+
+	// Needs to be less than 2x the wait time below, othewrwise the entry
+	// gets cleared out before it had a chance to be updated.
+	vfs.Opt.DirCacheTime = fs.Duration(50 * time.Millisecond)
+
+	r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
+
+	node, err := vfs.Stat("dir")
+	require.NoError(t, err)
+	modTime1 := node.(*Dir).DirEntry().ModTime(context.Background())
+
+	// Wait some time, then write another file which must update ModTime of
+	// the directory.
+	time.Sleep(75 * time.Millisecond)
+	r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2)
+
+	node2, err := vfs.Stat("dir")
+	require.NoError(t, err)
+	modTime2 := node2.(*Dir).DirEntry().ModTime(context.Background())
+
+	// ModTime of directory must be different after second file was written.
+	if modTime1.Equal(modTime2) {
+		t.Error("ModTime not invalidated")
+	}
+}