s3: Attempt to fix auth problems #5468

Revert "s3: remove WebIdentityRoleProvider to fix crash on auth #5255"

This reverts commit e618ea83dd.

VERSION

@@ -1 +1 @@
-v1.56.0
+v1.57.0

backend/s3/s3.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
@@ -629,6 +630,10 @@ func init() {
 				Value:    "s3.eu-central-1.wasabisys.com",
 				Help:     "Wasabi EU Central endpoint",
 				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ap-northeast-1.wasabisys.com",
+				Help:     "Wasabi AP Northeast endpoint",
+				Provider: "Wasabi",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -1541,6 +1546,11 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 			}),
 			ExpiryWindow: 3 * time.Minute,
 		},
+
+		// Pick up IAM role if we are in EKS
+		&stscreds.WebIdentityRoleProvider{
+			ExpiryWindow: 3 * time.Minute,
+		},
 	}
 	cred := credentials.NewChainCredentials(providers)
 

backend/sftp/sftp.go

@@ -313,13 +313,6 @@ type Object struct {
 	sha1sum *string     // Cached SHA1 checksum
 }
 
-// debugf calls fs.Debugf if --dump bodies or --dump headers is set
-func (f *Fs) debugf(o interface{}, text string, args ...interface{}) {
-	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
-		fs.Debugf(o, text, args...)
-	}
-}
-
 // dial starts a client connection to the given SSH server. It is a
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
@@ -771,9 +764,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 	if err != nil {
 		return nil, errors.Wrap(err, "NewFs")
 	}
-	f.debugf(f, "> Getwd")
 	cwd, err := c.sftpClient.Getwd()
-	f.debugf(f, "< Getwd: %q, err=%#v", cwd, err)
 	f.putSftpConnection(&c, nil)
 	if err != nil {
 		fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
@@ -854,9 +845,7 @@ func (f *Fs) dirExists(ctx context.Context, dir string) (bool, error) {
 	if err != nil {
 		return false, errors.Wrap(err, "dirExists")
 	}
-	f.debugf(f, "> Stat dirExists: %q", dir)
 	info, err := c.sftpClient.Stat(dir)
-	f.debugf(f, "< Stat dirExists: %#v, err=%#v", info, err)
 	f.putSftpConnection(&c, err)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -896,9 +885,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if err != nil {
 		return nil, errors.Wrap(err, "List")
 	}
-	f.debugf(f, "> ReadDir: %q", sftpDir)
 	infos, err := c.sftpClient.ReadDir(sftpDir)
-	f.debugf(f, "< ReadDir: %#v, err=%#v", infos, err)
 	f.putSftpConnection(&c, err)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error listing %q", dir)
@@ -989,9 +976,7 @@ func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
 	if err != nil {
 		return errors.Wrap(err, "mkdir")
 	}
-	f.debugf(f, "> Mkdir: %q", dirPath)
 	err = c.sftpClient.Mkdir(dirPath)
-	f.debugf(f, "< Mkdir: err=%#v", err)
 	f.putSftpConnection(&c, err)
 	if err != nil {
 		return errors.Wrapf(err, "mkdir %q failed", dirPath)
@@ -1022,9 +1007,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return errors.Wrap(err, "Rmdir")
 	}
-	f.debugf(f, "> Rmdir: %q", root)
 	err = c.sftpClient.RemoveDirectory(root)
-	f.debugf(f, "< Rmdir: err=%#v", err)
 	f.putSftpConnection(&c, err)
 	return err
 }
@@ -1044,10 +1027,10 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, errors.Wrap(err, "Move")
 	}
-	srcPath, dstPath := srcObj.path(), path.Join(f.absRoot, remote)
-	f.debugf(f, "> Rename file: src=%q, dst=%q", srcPath, dstPath)
-	err = c.sftpClient.Rename(srcPath, dstPath)
-	f.debugf(f, "< Rename file: err=%#v", err)
+	err = c.sftpClient.Rename(
+		srcObj.path(),
+		path.Join(f.absRoot, remote),
+	)
 	f.putSftpConnection(&c, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "Move Rename failed")
@@ -1096,12 +1079,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	if err != nil {
 		return errors.Wrap(err, "DirMove")
 	}
-	f.debugf(f, "> Rename dir: src=%q, dst=%q", srcPath, dstPath)
 	err = c.sftpClient.Rename(
 		srcPath,
 		dstPath,
 	)
-	f.debugf(f, "< Rename dir: err=%#v", err)
 	f.putSftpConnection(&c, err)
 	if err != nil {
 		return errors.Wrapf(err, "DirMove Rename(%q,%q) failed", srcPath, dstPath)
@@ -1117,9 +1098,7 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
 	}
 	defer f.putSftpConnection(&c, err)
 
-	f.debugf(f, "> NewSession run")
 	session, err := c.sshClient.NewSession()
-	f.debugf(f, "< NewSession run: %#v, err=%#v", session, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "run: get SFTP session")
 	}
@@ -1131,9 +1110,7 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
 	session.Stdout = &stdout
 	session.Stderr = &stderr
 
-	f.debugf(f, "> Run cmd: %q", cmd)
 	err = session.Run(cmd)
-	f.debugf(f, "< Run cmd: err=%#v", err)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to run %q: %s", cmd, stderr.Bytes())
 	}
@@ -1280,9 +1257,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "Hash get SFTP connection")
 	}
-	o.fs.debugf(o, "> NewSession hash")
 	session, err := c.sshClient.NewSession()
-	o.fs.debugf(o, "< NewSession hash: %#v, err=%#v", session, err)
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
 		return "", errors.Wrap(err, "Hash put SFTP connection")
@@ -1392,9 +1367,7 @@ func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err err
 		return nil, errors.Wrap(err, "stat")
 	}
 	absPath := path.Join(f.absRoot, remote)
-	f.debugf(f, "> Stat file: %q", absPath)
 	info, err = c.sftpClient.Stat(absPath)
-	f.debugf(f, "< Stat file: %#v, err=%#v", info, err)
 	f.putSftpConnection(&c, err)
 	return info, err
 }
@@ -1426,9 +1399,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if err != nil {
 		return errors.Wrap(err, "SetModTime")
 	}
-	o.fs.debugf(o, "> Chtimes: %q, %v", o.path(), modTime)
 	err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
-	o.fs.debugf(o, "< Chtimes: err=%#v", err)
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
 		return errors.Wrap(err, "SetModTime failed")
@@ -1516,9 +1487,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, errors.Wrap(err, "Open")
 	}
-	o.fs.debugf(o, "> Open read: %q", o.path())
 	sftpFile, err := c.sftpClient.Open(o.path())
-	o.fs.debugf(o, "< Open read: %#v, err=%#v", sftpFile, err)
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "Open failed")
@@ -1557,9 +1526,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return errors.Wrap(err, "Update")
 	}
-	o.fs.debugf(o, "> OpenFile write: %q", o.path())
 	file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
-	o.fs.debugf(o, "< OpenFile write: %#v, err=%#v", file, err)
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
 		return errors.Wrap(err, "Update Create failed")
@@ -1571,9 +1538,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			fs.Debugf(src, "Failed to open new SSH connection for delete: %v", removeErr)
 			return
 		}
-		o.fs.debugf(o, "> Remove file: %q", o.path())
 		removeErr = c.sftpClient.Remove(o.path())
-		o.fs.debugf(o, "< Remove file: err=%#v", removeErr)
 		o.fs.putSftpConnection(&c, removeErr)
 		if removeErr != nil {
 			fs.Debugf(src, "Failed to remove: %v", removeErr)
@@ -1622,9 +1587,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "Remove")
 	}
-	o.fs.debugf(o, "> Remove: %q", o.path())
 	err = c.sftpClient.Remove(o.path())
-	o.fs.debugf(o, "< Remove: err=%#v", err)
 	o.fs.putSftpConnection(&c, err)
 	return err
 }

docs/content/commands/rclone_mount.md

@@ -18,7 +18,7 @@ FUSE.
 
 First set up your remote using `rclone config`.  Check it works with `rclone ls` etc.
 
-On Linux and OSX, you can either run mount in foreground mode or background (daemon) mode.
+On Linux and macOS, you can either run mount in foreground mode or background (daemon) mode.
 Mount runs in foreground mode by default, use the `--daemon` flag to specify background mode.
 You can only run mount in foreground mode on Windows.
 
@@ -47,7 +47,7 @@ When running in background mode the user will have to stop the mount manually:
 
     # Linux
     fusermount -u /path/to/local/mount
-    # OS X
+    # macOS
     umount /path/to/local/mount
 
 The umount operation can fail, for example when the mountpoint is busy.
@@ -238,7 +238,7 @@ Hubic) do not support the concept of empty directories, so empty
 directories will have a tendency to disappear once they fall out of
 the directory cache.
 
-Only supported on Linux, FreeBSD, OS X and Windows at the moment.
+Only supported on Linux, FreeBSD, macOS and Windows at the moment.
 
 ## rclone mount vs rclone sync/copy
 
@@ -602,8 +602,8 @@ rclone mount remote:path /path/to/mountpoint [flags]
       --no-checksum                            Don't compare checksums on up/download.
       --no-modtime                             Don't read/write the modification time (can speed things up).
       --no-seek                                Don't allow seeking in files.
-      --noappledouble                          Ignore Apple Double (._) and .DS_Store files. Supported on OSX only. (default true)
-      --noapplexattr                           Ignore all "com.apple.*" extended attributes. Supported on OSX only.
+      --noappledouble                          Ignore Apple Double (._) and .DS_Store files. Supported on macOS only. (default true)
+      --noapplexattr                           Ignore all "com.apple.*" extended attributes. Supported on macOS only.
   -o, --option stringArray                     Option for libfuse/WinFsp. Repeat if required.
       --poll-interval duration                 Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable. (default 1m0s)
       --read-only                              Mount read-only.
@@ -621,7 +621,7 @@ rclone mount remote:path /path/to/mountpoint [flags]
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size.
       --vfs-write-back duration                Time to writeback files after last use when using cache. (default 5s)
       --vfs-write-wait duration                Time to wait for in-sequence write before giving error. (default 1s)
-      --volname string                         Set the volume name. Supported on Windows and OSX only.
+      --volname string                         Set the volume name. Supported on Windows and macOS only.
       --write-back-cache                       Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. Not supported on Windows.
 ```
 

docs/content/docs.md

@@ -910,6 +910,10 @@ While this isn't a generally recommended option, it can be useful
 in cases where your files change due to encryption. However, it cannot
 correct partial transfers in case a transfer was interrupted.
 
+When performing a `move`/`moveto` command, this flag will leave skipped
+files in the source location unchanged when a file with the same name
+exists on the destination.
+
 ### --ignore-size ###
 
 Normally rclone will look at modification time and size of files to

docs/content/drive.md

@@ -258,7 +258,7 @@ client_secret>   # Can be left blank
 scope>           # Select your scope, 1 for example
 root_folder_id>  # Can be left blank
 service_account_file> /home/foo/myJSONfile.json # This is where the JSON file goes!
-y/n>             # Auto config, y
+y/n>             # Auto config, n
 
 ```
 

docs/content/s3.md

@@ -1048,6 +1048,8 @@ Required when using an S3 clone.
         - Wasabi US West endpoint
     - "s3.eu-central-1.wasabisys.com"
         - Wasabi EU Central endpoint
+    - "s3.ap-northeast-1.wasabisys.com"
+        - Wasabi AP Northeast endpoint
 
 #### --s3-location-constraint
 

docs/content/yandex.md

@@ -124,6 +124,12 @@ to twice the max size of file in GiB should be enough, so if you want
 to upload a 30 GiB file set a timeout of `2 * 30 = 60m`, that is
 `--timeout 60m`.
 
+Having a Yandex Mail account is mandatory to use the Yandex.Disk subscription.
+Token generation will work without a mail account, but Rclone won't be able to complete any actions.
+```
+[403 - DiskUnsupportedUserAccountTypeError] User account type is not supported.
+```
+
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/yandex/yandex.go then run make backenddocs" >}}
 ### Standard Options
 

docs/layouts/partials/version.html

@@ -1 +1 @@
-v1.56.0
+v1.57.0

fs/operations/operations.go

@@ -1811,7 +1811,11 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 	} else {
 		tr := accounting.Stats(ctx).NewCheckingTransfer(srcObj)
 		if !cp {
-			err = DeleteFile(ctx, srcObj)
+			if ci.IgnoreExisting {
+				fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")
+			} else {
+				err = DeleteFile(ctx, srcObj)
+			}
 		}
 		tr.Done(ctx, err)
 	}

fs/operations/operations_test.go

@@ -811,6 +811,32 @@ func TestMoveFile(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file2)
 }
 
+func TestMoveFileWithIgnoreExisting(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	fstest.CheckItems(t, r.Flocal, file1)
+
+	ci.IgnoreExisting = true
+
+	err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.Flocal)
+	fstest.CheckItems(t, r.Fremote, file1)
+
+	// Recreate file with updated content
+	file1b := r.WriteFile("file1", "file1 modified", t2)
+	fstest.CheckItems(t, r.Flocal, file1b)
+
+	// Ensure modified file did not transfer and was not deleted
+	err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1b.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.Flocal, file1b)
+	fstest.CheckItems(t, r.Fremote, file1)
+}
+
 func TestCaseInsensitiveMoveFile(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)

fs/sync/sync.go

@@ -354,6 +354,8 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
 					// Delete src if no error on copy
 					if operations.SameObject(src, pair.Dst) {
 						fs.Logf(src, "Not removing source file as it is the same file as the destination")
+					} else if s.ci.IgnoreExisting {
+						fs.Debugf(src, "Not removing source file as destination file exists and --ignore-existing is set")
 					} else {
 						s.processError(operations.DeleteFile(s.ctx, src))
 					}

fs/sync/sync_test.go

@@ -1342,6 +1342,65 @@ func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file1, file2)
 }
 
+func TestMoveWithIgnoreExisting(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	file1 := r.WriteFile("existing", "potato", t1)
+	file2 := r.WriteFile("existing-b", "tomato", t1)
+
+	ci.IgnoreExisting = true
+
+	accounting.GlobalStats().ResetCounters()
+	err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)
+	require.NoError(t, err)
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Flocal,
+		[]fstest.Item{},
+		[]string{},
+		fs.GetModifyWindow(ctx, r.Flocal),
+	)
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		[]fstest.Item{
+			file1,
+			file2,
+		},
+		[]string{},
+		fs.GetModifyWindow(ctx, r.Fremote),
+	)
+
+	// Recreate first file with modified content
+	file1b := r.WriteFile("existing", "newpotatoes", t2)
+	accounting.GlobalStats().ResetCounters()
+	err = MoveDir(ctx, r.Fremote, r.Flocal, false, false)
+	require.NoError(t, err)
+	// Source items should still exist in modified state
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Flocal,
+		[]fstest.Item{
+			file1b,
+		},
+		[]string{},
+		fs.GetModifyWindow(ctx, r.Flocal),
+	)
+	// Dest items should not have changed
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		[]fstest.Item{
+			file1,
+			file2,
+		},
+		[]string{},
+		fs.GetModifyWindow(ctx, r.Fremote),
+	)
+}
+
 // Test a server-side move if possible, or the backup path if not
 func TestServerSideMove(t *testing.T) {
 	ctx := context.Background()

fs/version.go

@@ -1,4 +1,4 @@
 package fs
 
 // Version of rclone
-var Version = "v1.56.0-DEV"
+var Version = "v1.57.0-DEV"