union: make server side copies possible from outside the union

Before this change, only server side copies within the union would be
permitted.

This change allows server side copies from outside the union to
happen, delegating the checking to the individual upstream backend.

The same change should be made to Move and DirMove

Fixes #6287

backend/drive/drive.go

@@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.

backend/drive/drive_internal_test.go

@@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)

backend/local/local.go

@@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:             true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat

backend/local/local_internal_test.go

@@ -378,6 +378,9 @@ func TestFilter(t *testing.T) {
 	r.WriteFile("excluded", "excluded file", when)
 	f := r.Flocal.(*Fs)
 
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	// Add a filter
 	ctx, fi := filter.AddConfig(ctx)
 	require.NoError(t, fi.AddRule("+ included"))

backend/s3/s3.go

@@ -2758,8 +2758,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 		if isDirectory {
 			return nil
 		}
-		// compare the base name only since the listing will have a prefix
-		if path.Base(wantRemote) != path.Base(gotRemote) {
+		if wantRemote != gotRemote {
 			return nil
 		}
 		info = object

backend/s3/s3_internal_test.go

@@ -6,16 +6,12 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
-	"path"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -254,7 +250,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// Create an object
-	const fileName = "versions/test-versions.txt"
+	const fileName = "test-versions.txt"
 	contents := random.String(100)
 	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
@@ -284,7 +280,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 		}()
 
 		// Read the contents
-		entries, err := f.List(ctx, "versions")
+		entries, err := f.List(ctx, "")
 		require.NoError(t, err)
 		tests := 0
 		var fileNameVersion string
@@ -299,24 +295,12 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 				t.Run("ReadVersion", func(t *testing.T) {
 					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
 				})
-				t.Run("NewFs", func(t *testing.T) {
-					// Check we can find the object with NewFs
-					fPath := fs.ConfigString(f)
-					fPath = strings.Replace(fPath, ":", ",versions=true:", 1)
-					subFPath := fspath.JoinRootPath(fPath, entry.Remote())
-					subF, err := cache.Get(ctx, subFPath)
-					require.Equal(t, fs.ErrorIsFile, err, "Remote %q didn't find a file", subFPath)
-					require.NotNil(t, subF)
-					o, err := subF.NewObject(ctx, path.Base(entry.Remote()))
-					require.NoError(t, err)
-					assert.Equal(t, contents, fstests.ReadObject(ctx, t, o, -1))
-				})
 				assert.WithinDuration(t, obj.(*Object).lastModified, versionTime, time.Second, "object time must be with 1 second of version time")
 				fileNameVersion = remote
 				tests++
 			}
 		}
-		assert.Equal(t, 2, tests, "object missing from listing: %v", entries)
+		assert.Equal(t, 2, tests, "object missing from listing")
 
 		// Check we can read the object with a version suffix
 		t.Run("NewObject", func(t *testing.T) {

backend/union/union.go

@@ -222,19 +222,19 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't copy - not same remote type")
-		return nil, fs.ErrorCantCopy
-	}
-	o := srcObj.UnWrapUpstream()
-	su := o.UpstreamFs()
-	if su.Features().Copy == nil {
-		return nil, fs.ErrorCantCopy
+	var srcFs fs.Info
+	if srcObj, ok := src.(*Object); ok {
+		// Have a union object - unwrap
+		o := srcObj.UnWrapUpstream()
+		srcFs = o.UpstreamFs()
+		src = o
+	} else {
+		// Have a non union object - it might be compatible with a union member
+		srcFs = src.Fs()
 	}
 	var du *upstream.Fs
 	for _, u := range f.upstreams {
-		if operations.Same(u.RootFs, su.RootFs) {
+		if u.Features().Copy != nil && operations.Same(u.RootFs, srcFs) {
 			du = u
 		}
 	}
@@ -244,7 +244,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if !du.IsCreatable() {
 		return nil, fs.ErrorPermissionDenied
 	}
-	co, err := du.Features().Copy(ctx, o, remote)
+	co, err := du.Features().Copy(ctx, src, remote)
 	if err != nil || co == nil {
 		return nil, err
 	}

docs/content/docs.md

@@ -1868,13 +1868,22 @@ By default, rclone doesn't keep track of renamed files, so if you
 rename a file locally then sync it to a remote, rclone will delete the
 old file on the remote and upload a new copy.
 
-If you use this flag, and the remote supports server-side copy or
-server-side move, and the source and destination have a compatible
-hash, then this will track renames during `sync`
-operations and perform renaming server-side.
+An rclone sync with `--track-renames` runs like a normal sync, but keeps
+track of objects which exist in the destination but not in the source
+(which would normally be deleted), and which objects exist in the
+source but not the destination (which would normally be transferred).
+These objects are then candidates for renaming.
 
-Files will be matched by size and hash - if both match then a rename
-will be considered.
+After the sync, rclone matches up the source only and destination only
+objects using the `--track-renames-strategy` specified and either
+renames the destination object or transfers the source and deletes the
+destination object. `--track-renames` is stateless like all of
+rclone's syncs.
+
+To use this flag the destination must support server-side copy or
+server-side move, and to use a hash based `--track-renames-strategy`
+(the default) the source and the destination must have a compatible
+hash.
 
 If the destination does not support server-side copy or move, rclone
 will fall back to the default behaviour and log an error level message
@@ -1892,7 +1901,7 @@ Note also that `--track-renames` is incompatible with
 
 ### --track-renames-strategy (hash,modtime,leaf,size) ###
 
-This option changes the matching criteria for `--track-renames`.
+This option changes the file matching criteria for `--track-renames`.
 
 The matching is controlled by a comma separated selection of these tokens:
 
@@ -1901,15 +1910,15 @@ The matching is controlled by a comma separated selection of these tokens:
 - `leaf` - the name of the file not including its directory name
 - `size` - the size of the file (this is always enabled)
 
-So using `--track-renames-strategy modtime,leaf` would match files
+The default option is `hash`.
+
+Using `--track-renames-strategy modtime,leaf` would match files
 based on modification time, the leaf of the file name and the size
 only.
 
 Using `--track-renames-strategy modtime` or `leaf` can enable
 `--track-renames` support for encrypted destinations.
 
-If nothing is specified, the default option is matching by `hash`es.
-
 Note that the `hash` strategy is not supported with encrypted destinations.
 
 ### --delete-(before,during,after) ###

fs/features.go

@@ -29,6 +29,7 @@ type Features struct {
 	ReadMetadata            bool // can read metadata from objects
 	WriteMetadata           bool // can write metadata to objects
 	UserMetadata            bool // can read/write general purpose metadata
+	FilterAware             bool // can make use of filters if provided for listing
 
 	// Purge all files in the directory specified
 	//
@@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
 	// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
 	ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
 	ft.SlowHash = ft.SlowHash && mask.SlowHash
+	ft.FilterAware = ft.FilterAware && mask.FilterAware
 
 	if mask.Purge == nil {
 		ft.Purge = nil

fs/march/march.go

@@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
 	if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
 		!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
 		return func(dir string) (entries fs.DirEntries, err error) {
-			dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
+			dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 			return list.DirSorted(dirCtx, f, includeAll, dir)
 		}
 	}
@@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
 		mu.Lock()
 		defer mu.Unlock()
 		if !started {
-			dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
+			dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 			dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth)
 			started = true
 		}

fs/operations/operations.go

@@ -383,6 +383,20 @@ func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOptio
 	return hashType, &fs.HashesOption{Hashes: common}
 }
 
+// Is it OK to server side move/copy from src to dst
+func serverSideOK(ci *fs.ConfigInfo, fDst, fSrc fs.Info) bool {
+	if ci.ServerSideAcrossConfigs {
+		return true
+	}
+	if SameConfig(fSrc, fDst) {
+		return true
+	}
+	if SameRemoteType(fSrc, fDst) {
+		return fDst.Features().ServerSideAcrossConfigs
+	}
+	return false
+}
+
 // Copy src object to dst or f if nil.  If dst is nil then it uses
 // remote as the name of the new object.
 //
@@ -424,7 +438,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 				return nil, accounting.ErrorMaxTransferLimitReachedGraceful
 			}
 		}
-		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && (f.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
+		if doCopy := f.Features().Copy; doCopy != nil && serverSideOK(ci, f, src.Fs()) {
 			in := tr.Account(ctx, nil) // account the transfer
 			in.ServerSideCopyStart()
 			newDst, err = doCopy(ctx, src, remote)
@@ -619,7 +633,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 		return newDst, nil
 	}
 	// See if we have Move available
-	if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
+	if doMove := fdst.Features().Move; doMove != nil && serverSideOK(ci, fdst, src.Fs()) {
 		// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
 		if dst != nil && !SameObject(src, dst) {
 			err = DeleteFile(ctx, dst)

fs/walk/walk.go

@@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error
 func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
 	ci := fs.GetConfig(ctx)
 	fi := filter.GetConfig(ctx)
-	ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
+	ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 	if ci.NoTraverse && fi.HaveFilesFrom() {
 		return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject))
 	}
@@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel
 		fi.UsesDirectoryFilters() { // ...using any directory filters
 		return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
 	}
-	ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
+	ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 	return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
 }