vfs: make uploaded files retain modtime with non-modtime backends

Before this change if a file was uploaded to a backend which didn't
support modtimes, the time of the file read after the upload had
completed would change to the time the file was uploaded on the
backend.

When using `--vfs-cache-mode writes` or `full` this time would be
different by the `--vfs-write-back` delay which would cause
applications to think the file had been modified.

This changes uses the last modification time read in the cache as the
modtime for backends which don't support setting modtimes. It does not
change the modtime to that actually uploaded.

This means that as long as the file remains in the directory cache it
will have the expected modtime.

See: https://forum.rclone.org/t/saving-files-causes-wrong-modified-time-to-be-set-for-a-few-seconds-on-webdav-mount-with-bitrix24/36451

backend/drive/drive.go

@@ -761,7 +761,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
-			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
+			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
 				fs.Errorf(f, "Received upload limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {

backend/drive/drive_internal_test.go

@@ -243,6 +243,15 @@ func (f *Fs) InternalTestShouldRetry(t *testing.T) {
 	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
 	assert.False(t, quotaExceededRetry)
 	assert.Equal(t, quotaExceededError, expectedQuotaError)
+
+	sqEItem := googleapi.ErrorItem{
+		Reason: "storageQuotaExceeded",
+	}
+	generic403.Errors[0] = sqEItem
+	expectedStorageQuotaError := fserrors.FatalError(&generic403)
+	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, storageQuotaExceededRetry)
+	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
 }
 
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/ftp"
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -315,18 +315,26 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+// returns true if this FTP error should be retried
+func isRetriableFtpError(err error) bool {
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
+			return true
+		}
+	}
+	return false
+}
+
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
+	if isRetriableFtpError(err) {
+		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -1186,15 +1194,26 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+
+	var (
+		fd *ftp.Response
+		c  *ftp.ServerConn
+	)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		c, err = o.fs.getFtpConnection(ctx)
+		if err != nil {
+			return false, err // getFtpConnection has retries already
+		}
+		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+		if err != nil {
+			o.fs.putFtpConnection(&c, err)
+		}
+		return shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-	if err != nil {
-		o.fs.putFtpConnection(&c, err)
-		return nil, fmt.Errorf("open: %w", err)
-	}
+
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }

backend/googlecloudstorage/googlecloudstorage.go

@@ -82,7 +82,8 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			if saFile != "" || saCreds != "" || anonymous == "true" {
+			envAuth, _ := m.Get("env_auth")
+			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -330,6 +331,17 @@ can't check the size and hash but the file contents will be decompressed.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:    "env_auth",
+			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
+			Default: false,
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter credentials in the next step.",
+			}, {
+				Value: "true",
+				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
+			}},
 		}}...),
 	})
 }
@@ -349,6 +361,7 @@ type Options struct {
 	Decompress                bool                 `config:"decompress"`
 	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
+	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote storage server
@@ -500,6 +513,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
+	} else if opt.EnvAuth {
+		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
+		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {

backend/local/local.go

@@ -524,6 +524,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
+				// Quietly skip errors on excluded files and directories
+				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
+					continue
+				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))

backend/local/local_internal_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
@@ -395,3 +396,73 @@ func TestFilter(t *testing.T) {
 	sort.Sort(entries)
 	require.Equal(t, "[included]", fmt.Sprint(entries))
 }
+
+func TestFilterSymlink(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
+	r.WriteFile("included.file", "included file", when)
+	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
+	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
+	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
+	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
+
+	// Set fs into "-L" mode
+	f.opt.FollowSymlinks = true
+	f.opt.TranslateSymlinks = false
+	f.lstat = os.Stat
+
+	// Set fs into "-l" mode
+	// f.opt.FollowSymlinks = false
+	// f.opt.TranslateSymlinks = true
+	// f.lstat = os.Lstat
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Reset global error count
+	accounting.Stats(ctx).ResetErrors()
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included.file"))
+	require.NoError(t, fi.AddRule("+ included.file.link"))
+	require.NoError(t, fi.AddRule("+ included.dir/**"))
+	require.NoError(t, fi.AddRule("+ included.dir.link/**"))
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+
+	// Check 1 global errors one for each dangling symlink
+	assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+	accounting.Stats(ctx).ResetErrors()
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
+
+	// Check listing through a symlink still works
+	entries, err = f.List(ctx, "included.dir")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
+
+	sort.Sort(entries)
+	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
+}

backend/smb/connpool.go

@@ -34,9 +34,10 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{
 		Initiator: &smb2.NTLMInitiator{
-			User:     f.opt.User,
-			Password: pass,
-			Domain:   f.opt.Domain,
+			User:      f.opt.User,
+			Password:  pass,
+			Domain:    f.opt.Domain,
+			TargetSPN: f.opt.SPN,
 		},
 	}
 

backend/smb/smb.go

@@ -60,6 +60,17 @@ func init() {
 			Name:    "domain",
 			Help:    "Domain name for NTLM authentication.",
 			Default: "WORKGROUP",
+		}, {
+			Name: "spn",
+			Help: `Service principal name.
+
+Rclone presents this name to the server. Some servers use this as further
+authentication, and it often needs to be set for clusters. For example:
+
+    cifs/remotehost:1020
+
+Leave blank if not sure.
+`,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -109,6 +120,7 @@ type Options struct {
 	User            string      `config:"user"`
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
+	SPN             string      `config:"spn"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`

cmd/cmount/mountpoint_other.go

@@ -26,6 +26,5 @@ func getMountpoint(f fs.Fs, mountPath string, opt *mountlib.Options) (string, er
 	if err = mountlib.CheckAllowNonEmpty(mountPath, opt); err != nil {
 		return "", err
 	}
-	opt.VolumeName = mountlib.MakeVolumeNameValidOnUnix(opt.VolumeName)
 	return mountPath, nil
 }

cmd/cmount/mountpoint_windows.go

@@ -9,11 +9,9 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"strings"
 
 	"github.com/rclone/rclone/cmd/mountlib"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
 )
 
@@ -21,13 +19,10 @@ var isDriveRegex = regexp.MustCompile(`^[a-zA-Z]\:$`)
 var isDriveRootPathRegex = regexp.MustCompile(`^[a-zA-Z]\:\\$`)
 var isDriveOrRootPathRegex = regexp.MustCompile(`^[a-zA-Z]\:\\?$`)
 var isNetworkSharePathRegex = regexp.MustCompile(`^\\\\[^\\\?]+\\[^\\]`)
-var isAnyPathSeparatorRegex = regexp.MustCompile(`[/\\]+`) // Matches any path separators, slash or backslash, or sequences of them
 
-// isNetworkSharePath returns true if the given string is a network share path,
-// in the basic UNC format "\\Server\Share\Path". The first two path components
-// are required ("\\Server\Share"), and represents the volume. The rest of the
-// string can be anything, i.e. can be a nested path ("\\Server\Share\Path\Path\Path").
-// Actual validity of the path, e.g. if it contains invalid characters, is not considered.
+// isNetworkSharePath returns true if the given string is a valid network share path,
+// in the basic UNC format "\\Server\Share\Path", where the first two path components
+// are required ("\\Server\Share", which represents the volume).
 // Extended-length UNC format "\\?\UNC\Server\Share\Path" is not considered, as it is
 // not supported by cgofuse/winfsp, so returns false for any paths with prefix "\\?\".
 // Note: There is a UNCPath function in lib/file, but it refers to any extended-length
@@ -116,7 +111,7 @@ func handleLocalMountpath(f fs.Fs, mountpath string, opt *mountlib.Options) (str
 		// Drive letter string can be used as is, since we have already checked it does not exist,
 		// but directory path needs more checks.
 		if opt.NetworkMode {
-			fs.Debugf(nil, "Ignoring --network-mode as it is not supported with directory mountpoint")
+			fs.Errorf(nil, "Ignoring --network-mode as it is not supported with directory mountpoint")
 			opt.NetworkMode = false
 		}
 		var err error
@@ -137,47 +132,30 @@ func handleLocalMountpath(f fs.Fs, mountpath string, opt *mountlib.Options) (str
 	return mountpath, nil
 }
 
-// networkSharePathEncoder is an encoder used to make strings valid as (part of) Windows network share UNC paths
-const networkSharePathEncoder = (encoder.EncodeZero | // NUL(0x00)
-	encoder.EncodeCtl | // CTRL(0x01-0x1F)
-	encoder.EncodeDel | // DEL(0x7F)
-	encoder.EncodeWin | // :?"*<>|
-	encoder.EncodeInvalidUtf8) // Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
-
-// encodeNetworkSharePath makes a string valid to use as (part of) a Windows network share UNC path.
-// Using backslash as path separator here, but forward slashes would also be treated as
-// path separators by the library, and therefore does not encode either of them. For convenience,
-// normalizes to backslashes-only. UNC paths always start with two path separators, but WinFsp
-// requires volume prefix as UNC-like path but with only a single backslash prefix, and multiple
-// separators are not valid in any other parts of network share paths, so therefore (unlike what
-// filepath.FromSlash would do) replaces multiple separators with a single one (like filpath.Clean
-// would do, but it does also more). A trailing path separator would just be ignored, but we
-// remove it here as well for convenience.
-func encodeNetworkSharePath(volumeName string) string {
-	return networkSharePathEncoder.Encode(strings.TrimRight(isAnyPathSeparatorRegex.ReplaceAllString(volumeName, `\`), `\`))
-}
-
 // handleVolumeName handles the volume name option.
-func handleVolumeName(opt *mountlib.Options) {
-	// Ensure the volume name option is a valid network share UNC path if network mode,
+func handleVolumeName(opt *mountlib.Options, volumeName string) {
+	// If volumeName parameter is set, then just set that into options replacing any existing value.
+	// Else, ensure the volume name option is a valid network share UNC path if network mode,
 	// and ensure network mode if configured volume name is already UNC path.
-	if opt.VolumeName != "" { // Should always be true due to code in mountlib caller
+	if volumeName != "" {
+		opt.VolumeName = volumeName
+	} else if opt.VolumeName != "" { // Should always be true due to code in mountlib caller
 		// Use value of given volume name option, but check if it is disk volume name or network volume prefix
 		if isNetworkSharePath(opt.VolumeName) {
 			// Specified volume name is network share UNC path, assume network mode and use it as volume prefix
-			opt.VolumeName = encodeNetworkSharePath(opt.VolumeName[1:]) // We know from isNetworkSharePath it has a duplicate path separator prefix, so removes that right away (but encodeNetworkSharePath would remove it also)
+			opt.VolumeName = opt.VolumeName[1:] // WinFsp requires volume prefix as UNC-like path but with only a single backslash
 			if !opt.NetworkMode {
 				// Specified volume name is network share UNC path, force network mode and use it as volume prefix
 				fs.Debugf(nil, "Forcing network mode due to network share (UNC) volume name")
 				opt.NetworkMode = true
 			}
 		} else if opt.NetworkMode {
-			// Specified volume name is not a valid network share UNC path, but network mode is enabled, so append to a hard coded server prefix and use it as volume prefix
-			opt.VolumeName = `\server\` + strings.TrimLeft(encodeNetworkSharePath(opt.VolumeName), `\`)
+			// Plain volume name treated as share name in network mode, append to hard coded "\\server" prefix to get full volume prefix.
+			opt.VolumeName = "\\server\\" + opt.VolumeName
 		}
 	} else if opt.NetworkMode {
-		// Use hard coded default
-		opt.VolumeName = `\server\share`
+		// Hard coded default
+		opt.VolumeName = "\\server\\share"
 	}
 }
 
@@ -196,27 +174,22 @@ func getMountpoint(f fs.Fs, mountpath string, opt *mountlib.Options) (mountpoint
 	}
 
 	// Handle mountpath
+	var volumeName string
 	if isDefaultPath(mountpath) {
 		// Mount path indicates defaults, which will automatically pick an unused drive letter.
-		if mountpoint, err = handleDefaultMountpath(); err != nil {
-			return
-		}
+		mountpoint, err = handleDefaultMountpath()
 	} else if isNetworkSharePath(mountpath) {
 		// Mount path is a valid network share path (UNC format, "\\Server\Share" prefix).
-		if mountpoint, err = handleNetworkShareMountpath(mountpath, opt); err != nil {
-			return
-		}
-		// In this case the volume name is taken from the mount path, it replaces any existing volume name option.
-		opt.VolumeName = mountpath
+		mountpoint, err = handleNetworkShareMountpath(mountpath, opt)
+		// In this case the volume name is taken from the mount path, will replace any existing volume name option.
+		volumeName = mountpath[1:] // WinFsp requires volume prefix as UNC-like path but with only a single backslash
 	} else {
 		// Mount path is drive letter or directory path.
-		if mountpoint, err = handleLocalMountpath(f, mountpath, opt); err != nil {
-			return
-		}
+		mountpoint, err = handleLocalMountpath(f, mountpath, opt)
 	}
 
 	// Handle volume name
-	handleVolumeName(opt)
+	handleVolumeName(opt, volumeName)
 
 	// Done, return mountpoint to be used, together with updated mount options.
 	if opt.NetworkMode {

cmd/mount/mount.go

@@ -79,7 +79,6 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error
 	if err := mountlib.CheckAllowNonEmpty(mountpoint, opt); err != nil {
 		return nil, nil, err
 	}
-	opt.VolumeName = mountlib.MakeVolumeNameValidOnUnix(opt.VolumeName)
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
 	if opt.DebugFUSE {

cmd/mount2/mount.go

@@ -151,7 +151,6 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error
 	if err := mountlib.CheckAllowNonEmpty(mountpoint, opt); err != nil {
 		return nil, nil, err
 	}
-	opt.VolumeName = mountlib.MakeVolumeNameValidOnUnix(opt.VolumeName)
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
 	fsys := NewFS(VFS, opt)

cmd/mountlib/mount.go

@@ -57,7 +57,6 @@ var DefaultOpt = Options{
 	NoAppleDouble: true,            // use noappledouble by default
 	NoAppleXattr:  false,           // do not use noapplexattr by default
 	AsyncRead:     true,            // do async reads by default
-	NetworkMode:   true,            // use network mode by default (Windows only)
 }
 
 type (
@@ -240,12 +239,8 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
 func (m *MountPoint) Mount() (daemon *os.Process, err error) {
 
 	// Ensure sensible defaults
-	if m.MountOpt.VolumeName == "" {
-		m.MountOpt.VolumeName = fs.ConfigString(m.Fs)
-	}
-	if m.MountOpt.DeviceName == "" {
-		m.MountOpt.DeviceName = fs.ConfigString(m.Fs)
-	}
+	m.SetVolumeName(m.MountOpt.VolumeName)
+	m.SetDeviceName(m.MountOpt.DeviceName)
 
 	// Start background task if --daemon is specified
 	if m.MountOpt.Daemon {

cmd/mountlib/utils.go

@@ -97,10 +97,29 @@ func checkMountEmpty(mountpoint string) error {
 	return fmt.Errorf(msg+": %w", mountpoint, err)
 }
 
-// MakeVolumeNameValidOnUnix takes a volume name and returns a variant that is valid on unix systems.
-func MakeVolumeNameValidOnUnix(volumeName string) string {
-	volumeName = strings.ReplaceAll(volumeName, ":", " ")
-	volumeName = strings.ReplaceAll(volumeName, "/", " ")
-	volumeName = strings.TrimSpace(volumeName)
-	return volumeName
+// SetVolumeName with sensible default
+func (m *MountPoint) SetVolumeName(vol string) {
+	if vol == "" {
+		vol = fs.ConfigString(m.Fs)
+	}
+	m.MountOpt.SetVolumeName(vol)
+}
+
+// SetVolumeName removes special characters from volume name if necessary
+func (o *Options) SetVolumeName(vol string) {
+	vol = strings.ReplaceAll(vol, ":", " ")
+	vol = strings.ReplaceAll(vol, "/", " ")
+	vol = strings.TrimSpace(vol)
+	if runtime.GOOS == "windows" && len(vol) > 32 {
+		vol = vol[:32]
+	}
+	o.VolumeName = vol
+}
+
+// SetDeviceName with sensible default
+func (m *MountPoint) SetDeviceName(dev string) {
+	if dev == "" {
+		dev = fs.ConfigString(m.Fs)
+	}
+	m.MountOpt.DeviceName = dev
 }

cmd/tree/tree.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs/dirtree"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/terminal"
 	"github.com/spf13/cobra"
 )
@@ -27,6 +28,7 @@ var (
 	outFileName string
 	noReport    bool
 	sort        string
+	enc         = encoder.OS
 )
 
 func init() {
@@ -163,7 +165,7 @@ type FileInfo struct {
 
 // Name is base name of the file
 func (to *FileInfo) Name() string {
-	return path.Base(to.entry.Remote())
+	return enc.FromStandardName(path.Base(to.entry.Remote()))
 }
 
 // Size in bytes for regular files; system-dependent for others
@@ -197,7 +199,7 @@ func (to *FileInfo) Sys() interface{} {
 
 // String returns the full path
 func (to *FileInfo) String() string {
-	return to.entry.Remote()
+	return filepath.FromSlash(enc.FromStandardPath(to.entry.Remote()))
 }
 
 // Fs maps an fs.Fs into a tree.Fs
@@ -212,6 +214,7 @@ func NewFs(dirs dirtree.DirTree) Fs {
 func (dirs Fs) Stat(filePath string) (fi os.FileInfo, err error) {
 	defer log.Trace(nil, "filePath=%q", filePath)("fi=%+v, err=%v", &fi, &err)
 	filePath = filepath.ToSlash(filePath)
+	filePath = enc.ToStandardPath(filePath)
 	filePath = strings.TrimLeft(filePath, "/")
 	if filePath == "" {
 		return &FileInfo{fs.NewDir("", time.Now())}, nil
@@ -227,13 +230,14 @@ func (dirs Fs) Stat(filePath string) (fi os.FileInfo, err error) {
 func (dirs Fs) ReadDir(dir string) (names []string, err error) {
 	defer log.Trace(nil, "dir=%s", dir)("names=%+v, err=%v", &names, &err)
 	dir = filepath.ToSlash(dir)
+	dir = enc.ToStandardPath(dir)
 	dir = strings.TrimLeft(dir, "/")
 	entries, ok := dirs[dir]
 	if !ok {
 		return nil, fmt.Errorf("couldn't find directory %q", dir)
 	}
 	for _, entry := range entries {
-		names = append(names, path.Base(entry.Remote()))
+		names = append(names, enc.FromStandardName(path.Base(entry.Remote())))
 	}
 	return
 }

docs/content/authors.md

@@ -689,3 +689,7 @@ put them back in again.` >}}
   * Gerard Bosch <30733556+gerardbosch@users.noreply.github.com>
   * ToBeFree <github@tfrei.de>
   * NodudeWasTaken <75137537+NodudeWasTaken@users.noreply.github.com>
+  * Peter Brunner <peter@lugoues.net>
+  * Ninh Pham <dongian.rapclubkhtn@gmail.com>
+  * Ryan Caezar Itang <sitiom@proton.me>
+  * Peter Brunner <peter@psykhe.com>

docs/content/install.md

@@ -165,6 +165,19 @@ developers so it may be out of date. Its current version is as below.
 
 [![Chocolatey package](https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)](https://repology.org/project/rclone/versions)
 
+### Scoop package manager {#windows-scoop}
+
+Make sure you have [Scoop](https://scoop.sh/) installed
+
+```
+scoop install rclone
+```
+
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[![Scoop package](https://repology.org/badge/version-for-repo/scoop/rclone.svg)](https://repology.org/project/rclone/versions)
+
 ## Package manager installation {#package-manager}
 
 Many Linux, Windows, macOS and other OS distributions package and

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/iguanesolutions/go-systemd/v5 v5.1.0
 	github.com/jcmturner/gokrb5/v8 v8.4.3
+	github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
 	github.com/klauspost/compress v1.15.14
 	github.com/koofr/go-httpclient v0.0.0-20221124135700-2eb26cff5dd8
@@ -47,7 +48,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
-	github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6
 	github.com/rfjakob/eme v1.1.2
 	github.com/rivo/uniseg v0.4.3
 	github.com/shirou/gopsutil/v3 v3.22.12
@@ -62,7 +62,7 @@ require (
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0
 	go.etcd.io/bbolt v1.3.6
-	goftp.io/server v0.4.1
+	goftp.io/server v0.4.2-0.20210615155358-d07a820aac35
 	golang.org/x/crypto v0.5.0
 	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.4.0

go.sum

@@ -277,8 +277,9 @@ github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE
 github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf h1:2IYBd5TD/maMqTU2YUzp2tJL4cNaOYQ9EBullN9t9pk=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
+github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e h1:Xofa5zcfulLjSb9ZNpb7MI9TFCpVkPCy3JSwrL7xoWE=
+github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e/go.mod h1:sRSt+7UoQ5BgrZhwta4kr7N5SenQsoIZHMJHY7+zqJg=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -407,8 +408,6 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 h1:J832KfU2Z44Ck3XR5bvw2UxShP0QnjueruNQ6dTYH+g=
-github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8=
 github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
 github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -506,8 +505,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-goftp.io/server v0.4.1 h1:x7KG4HIxSMdK/rpYhExMinRN/aO/T9icvaG/B5e/XfY=
-goftp.io/server v0.4.1/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k=
+goftp.io/server v0.4.2-0.20210615155358-d07a820aac35 h1:D4DhKKOtievTsshtbA6W0XL/gBjERF5/vu6Vhmb4sBw=
+goftp.io/server v0.4.2-0.20210615155358-d07a820aac35/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

vfs/dir.go

@@ -325,10 +325,15 @@ func (d *Dir) renameTree(dirPath string) {
 		d.entry = fs.NewDirCopy(context.TODO(), d.entry).SetRemote(dirPath)
 	}
 
-	// Do the same to any child directories
+	// Do the same to any child directories and files
 	for leaf, node := range d.items {
-		if dir, ok := node.(*Dir); ok {
-			dir.renameTree(path.Join(dirPath, leaf))
+		switch x := node.(type) {
+		case *Dir:
+			x.renameTree(path.Join(dirPath, leaf))
+		case *File:
+			x.renameDir(dirPath)
+		default:
+			panic("bad dir entry")
 		}
 	}
 }

vfs/dir_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"testing"
 	"time"
+	"unsafe"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
@@ -577,3 +578,7 @@ func TestDirRename(t *testing.T) {
 	err = dir.Rename("potato", "tuba", dir)
 	assert.Equal(t, EROFS, err)
 }
+
+func TestDirStructSize(t *testing.T) {
+	t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{}))
+}

vfs/file.go

@@ -141,6 +141,13 @@ func (f *File) Node() Node {
 	return f
 }
 
+// renameDir - call when parent directory has been renamed
+func (f *File) renameDir(dPath string) {
+	f.mu.RLock()
+	f.dPath = dPath
+	f.mu.RUnlock()
+}
+
 // applyPendingRename runs a previously set rename operation if there are no
 // more remaining writers. Call without lock held.
 func (f *File) applyPendingRename() {
@@ -296,6 +303,9 @@ func (f *File) activeWriters() int {
 // It should be called with the lock held
 func (f *File) _roundModTime(modTime time.Time) time.Time {
 	precision := f.d.f.Precision()
+	if precision == fs.ModTimeNotSupported {
+		return modTime
+	}
 	return modTime.Truncate(precision)
 }
 
@@ -312,7 +322,9 @@ func (f *File) ModTime() (modTime time.Time) {
 	}
 	// Read the modtime from a dirty item if it exists
 	if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
-		if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
+		item := f.d.vfs.cache.ItemOrNil(f._path())
+		noModTime := f.d.f.Precision() == fs.ModTimeNotSupported
+		if item != nil && (item.IsDirty() || noModTime) {
 			modTime, err := item.GetModTime()
 			if err != nil {
 				fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)

vfs/file_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"testing"
+	"unsafe"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
@@ -411,3 +412,7 @@ func TestFileRename(t *testing.T) {
 		})
 	}
 }
+
+func TestFileStructSize(t *testing.T) {
+	t.Logf("File struct has size %d bytes", unsafe.Sizeof(File{}))
+}

vfs/vfscache/cache.go

@@ -294,14 +294,22 @@ func (c *Cache) put(name string, item *Item) (oldItem *Item) {
 	return oldItem
 }
 
+// ItemOrNil returns the Item if it exists in the cache otherwise it
+// returns nil.
+//
+// name should be a remote path not an osPath
+func (c *Cache) ItemOrNil(name string) (item *Item) {
+	name = clean(name)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.item[name]
+}
+
 // InUse returns whether the name is in use in the cache
 //
 // name should be a remote path not an osPath
 func (c *Cache) InUse(name string) bool {
-	name = clean(name)
-	c.mu.Lock()
-	item := c.item[name]
-	c.mu.Unlock()
+	item := c.ItemOrNil(name)
 	if item == nil {
 		return false
 	}
@@ -313,10 +321,7 @@ func (c *Cache) InUse(name string) bool {
 //
 // name should be a remote path not an osPath
 func (c *Cache) DirtyItem(name string) (item *Item) {
-	name = clean(name)
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	item = c.item[name]
+	item = c.ItemOrNil(name)
 	if item != nil && !item.IsDirty() {
 		item = nil
 	}

vfs/vfscache/item.go

@@ -1223,7 +1223,7 @@ func (item *Item) setModTime(modTime time.Time) {
 	item.mu.Unlock()
 }
 
-// GetModTime of the cache file
+// GetModTime of the cache item
 func (item *Item) GetModTime() (modTime time.Time, err error) {
 	// defer log.Trace(item.name, "modTime=%v", modTime)("")
 	item.mu.Lock()
@@ -1231,6 +1231,9 @@ func (item *Item) GetModTime() (modTime time.Time, err error) {
 	fi, err := item._stat()
 	if err == nil {
 		modTime = fi.ModTime()
+		item.info.ModTime = modTime
+	} else {
+		modTime = item.info.ModTime
 	}
 	return modTime, nil
 }