fs: dump config for an Fs before creation - FIXME TESTING DO NOT MERGE

See: https://forum.rclone.org/t/authenticating-to-box-using-connection-strings-and-config-file-fails/23333/7

.github/ISSUE_TEMPLATE/Bug.md

@@ -5,19 +5,31 @@ about: Report a problem with rclone
 
 <!--
 
-Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
+We understand you are having a problem with rclone; we want to help you with that!
 
-If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
+**STOP and READ**
+**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
+Please show the effort you've put in to solving the problem and please be specific.
+People are volunteering their time to help! Low effort posts are not likely to get good answers!
+
+If you think you might have found a bug, try to replicate it with the latest beta (or stable).
+The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
+
+If you can still replicate it or just got a question then please use the rclone forum:
 
     https://forum.rclone.org/
 
-instead of filing an issue for a quick response.
+for a quick response instead of filing an issue on this repo.
 
-If you think you might have found a bug, please can you try to replicate it with the latest beta?
+If nothing else helps, then please fill in the info below which helps us help you.
 
-    https://beta.rclone.org/
+**DO NOT REDACT** any information except passwords/keys/personal info.
 
-If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
+You should use 3 backticks to begin and end your paste to make it readable.
+
+Make sure to include a log obtained with '-vv'.
+
+You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
 
 Thank you
 
@@ -25,6 +37,11 @@ The Rclone Developers
 
 -->
 
+
+#### The associated forum post URL from `https://forum.rclone.org`
+
+
+
 #### What is the problem you are having with rclone?
 
 

.github/ISSUE_TEMPLATE/Feature.md

@@ -7,12 +7,16 @@ about: Suggest a new feature or enhancement for rclone
 
 Welcome :-)
 
-So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
+So you've got an idea to improve rclone? We love that!
+You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
 
-Here is a checklist of things to do:
+Probably the latest beta (or stable) release has your feature, so try to update your rclone.
+The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
 
-  1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum first: https://forum.rclone.org/
+If it still isn't there, here is a checklist of things to do:
+
+  1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
+  2. Discuss on the forum: https://forum.rclone.org/
   3. Make a feature request issue (this is the right place!).
   4. Be prepared to get involved making the feature :-)
 
@@ -23,6 +27,10 @@ The Rclone Developers
 -->
 
 
+#### The associated forum post URL from `https://forum.rclone.org`
+
+
+
 #### What is your current rclone version (output from `rclone version`)?
 
 

CONTRIBUTING.md

@@ -33,10 +33,11 @@ page](https://github.com/rclone/rclone).
 
 Now in your terminal
 
-    go get -u github.com/rclone/rclone
-    cd $GOPATH/src/github.com/rclone/rclone
+    git clone https://github.com/rclone/rclone.git
+    cd rclone
     git remote rename origin upstream
     git remote add origin git@github.com:YOURUSER/rclone.git
+    go build
 
 Make a branch to add your new feature
 

VERSION

@@ -1 +1 @@
-v1.55.0
+v1.56.0

backend/drive/drive.go

@@ -2959,12 +2959,12 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 }
 
 // List all team drives
-func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err error) {
-	drives = []*drive.TeamDrive{}
-	listTeamDrives := f.svc.Teamdrives.List().PageSize(100)
+func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.Drive, err error) {
+	drives = []*drive.Drive{}
+	listTeamDrives := f.svc.Drives.List().PageSize(100)
 	var defaultFs Fs // default Fs with default Options
 	for {
-		var teamDrives *drive.TeamDriveList
+		var teamDrives *drive.DriveList
 		err = f.pacer.Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Context(ctx).Do()
 			return defaultFs.shouldRetry(ctx, err)
@@ -2972,7 +2972,7 @@ func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err
 		if err != nil {
 			return drives, errors.Wrap(err, "listing Team Drives failed")
 		}
-		drives = append(drives, teamDrives.TeamDrives...)
+		drives = append(drives, teamDrives.Drives...)
 		if teamDrives.NextPageToken == "" {
 			break
 		}
@@ -3069,7 +3069,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 		return err
 	}
 	if destLeaf == "" {
-		destLeaf = info.Name
+		destLeaf = path.Base(o.Remote())
 	}
 	if destDir == "" {
 		destDir = "."

backend/dropbox/dropbox.go

@@ -1084,13 +1084,30 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
-		// FIXME this gives settings_error/not_authorized/.. errors
-		// and the expires setting isn't in the documentation so remove
-		// for now.
-		// Settings: &sharing.SharedLinkSettings{
-		// 	Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
-		// },
+		Settings: &sharing.SharedLinkSettings{
+			RequestedVisibility: &sharing.RequestedVisibility{
+				Tagged: dropbox.Tagged{Tag: sharing.RequestedVisibilityPublic},
+			},
+			Audience: &sharing.LinkAudience{
+				Tagged: dropbox.Tagged{Tag: sharing.LinkAudiencePublic},
+			},
+			Access: &sharing.RequestedLinkAccessLevel{
+				Tagged: dropbox.Tagged{Tag: sharing.RequestedLinkAccessLevelViewer},
+			},
+		},
 	}
+	if expire < fs.DurationOff {
+		expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
+		createArg.Settings.Expires = expiryTime
+	}
+	// FIXME note we can't set Settings for non enterprise dropbox
+	// because of https://github.com/dropbox/dropbox-sdk-go-unofficial/issues/75
+	// however this only goes wrong when we set Expires, so as a
+	// work-around remove Settings unless expire is set.
+	if expire == fs.DurationOff {
+		createArg.Settings = nil
+	}
+
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
 		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)

backend/fichier/fichier.go

@@ -348,8 +348,10 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		return nil, err
 	}
 
-	if len(fileUploadResponse.Links) != 1 {
-		return nil, errors.New("unexpected amount of files")
+	if len(fileUploadResponse.Links) == 0 {
+		return nil, errors.New("upload response not found")
+	} else if len(fileUploadResponse.Links) > 1 {
+		fs.Debugf(remote, "Multiple upload responses found, using the first")
 	}
 
 	link := fileUploadResponse.Links[0]

backend/onedrive/onedrive.go

@@ -361,6 +361,11 @@ This will only work if you are copying between two OneDrive *Personal* drives AN
 the files to copy are already shared between them.  In other cases, rclone will
 fall back to normal copy (which will be slightly slower).`,
 			Advanced: true,
+		}, {
+			Name:     "list_chunk",
+			Help:     "Size of listing chunk.",
+			Default:  1000,
+			Advanced: true,
 		}, {
 			Name:    "no_versions",
 			Default: false,
@@ -468,6 +473,7 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
+	ListChunk               int64                `config:"list_chunk"`
 	NoVersions              bool                 `config:"no_versions"`
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
@@ -896,7 +902,7 @@ type listAllFn func(*api.Item) bool
 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	// Top parameter asks for bigger pages of data
 	// https://dev.onedrive.com/odata/optional-query-parameters.htm
-	opts := f.newOptsCall(dirID, "GET", "/children?$top=1000")
+	opts := f.newOptsCall(dirID, "GET", fmt.Sprintf("/children?$top=%d", f.opt.ListChunk))
 OUTER:
 	for {
 		var result api.ListChildrenResponse
@@ -1423,7 +1429,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		Password: f.opt.LinkPassword,
 	}
 
-	if expire < fs.Duration(time.Hour*24*365*100) {
+	if expire < fs.DurationOff {
 		expiry := time.Now().Add(time.Duration(expire))
 		share.Expiry = &expiry
 	}

backend/sftp/sftp.go

@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -286,6 +287,7 @@ type Fs struct {
 	drain        *time.Timer // used to drain the pool when we stop using the connections
 	pacer        *fs.Pacer   // pacer for operations
 	savedpswd    string
+	transfers    int32 // count in use references
 }
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -348,6 +350,23 @@ func (c *conn) closed() error {
 	return nil
 }
 
+// Show that we are doing an upload or download
+//
+// Call removeTransfer() when done
+func (f *Fs) addTransfer() {
+	atomic.AddInt32(&f.transfers, 1)
+}
+
+// Show the upload or download done
+func (f *Fs) removeTransfer() {
+	atomic.AddInt32(&f.transfers, -1)
+}
+
+// getTransfers shows whether there are any transfers in progress
+func (f *Fs) getTransfers() int32 {
+	return atomic.LoadInt32(&f.transfers)
+}
+
 // Open a new connection to the SFTP server.
 func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
 	// Rate limit rate of new connections
@@ -395,8 +414,12 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
 	opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
 	opts = append(opts,
 		sftp.UseFstat(f.opt.UseFstat),
-		sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
+		// FIXME disabled after library reversion
+		// sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
 	)
+	if f.opt.DisableConcurrentReads { // FIXME
+		fs.Errorf(f, "Ignoring disable_concurrent_reads after library reversion - see #5197")
+	}
 
 	return sftp.NewClientPipe(pr, pw, opts...)
 }
@@ -474,6 +497,13 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 func (f *Fs) drainPool(ctx context.Context) (err error) {
 	f.poolMu.Lock()
 	defer f.poolMu.Unlock()
+	if transfers := f.getTransfers(); transfers != 0 {
+		fs.Debugf(f, "Not closing %d unused connections as %d transfers in progress", len(f.pool), transfers)
+		if f.opt.IdleTimeout > 0 {
+			f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+		}
+		return nil
+	}
 	if f.opt.IdleTimeout > 0 {
 		f.drain.Stop()
 	}
@@ -1380,18 +1410,22 @@ func (o *Object) Storable() bool {
 
 // objectReader represents a file open for reading on the SFTP server
 type objectReader struct {
+	f          *Fs
 	sftpFile   *sftp.File
 	pipeReader *io.PipeReader
 	done       chan struct{}
 }
 
-func newObjectReader(sftpFile *sftp.File) *objectReader {
+func (f *Fs) newObjectReader(sftpFile *sftp.File) *objectReader {
 	pipeReader, pipeWriter := io.Pipe()
 	file := &objectReader{
+		f:          f,
 		sftpFile:   sftpFile,
 		pipeReader: pipeReader,
 		done:       make(chan struct{}),
 	}
+	// Show connection in use
+	f.addTransfer()
 
 	go func() {
 		// Use sftpFile.WriteTo to pump data so that it gets a
@@ -1421,6 +1455,8 @@ func (file *objectReader) Close() (err error) {
 	_ = file.pipeReader.Close()
 	// Wait for the background process to finish
 	<-file.done
+	// Show connection no longer in use
+	file.f.removeTransfer()
 	return err
 }
 
@@ -1454,12 +1490,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			return nil, errors.Wrap(err, "Open Seek failed")
 		}
 	}
-	in = readers.NewLimitedReadCloser(newObjectReader(sftpFile), limit)
+	in = readers.NewLimitedReadCloser(o.fs.newObjectReader(sftpFile), limit)
 	return in, nil
 }
 
 // Update a remote sftp file using the data <in> and ModTime from <src>
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	o.fs.addTransfer() // Show transfer in progress
+	defer o.fs.removeTransfer()
 	// Clear the hash cache since we are about to update the object
 	o.md5sum = nil
 	o.sha1sum = nil

backend/zoho/zoho.go

@@ -96,6 +96,11 @@ func init() {
 					log.Fatalf("Failed to configure token: %v", err)
 				}
 			}
+
+			if fs.GetConfig(ctx).AutoConfirm {
+				return
+			}
+
 			if err = setupRoot(ctx, name, m); err != nil {
 				log.Fatalf("Failed to configure root directory: %v", err)
 			}
@@ -161,7 +166,7 @@ type Object struct {
 
 func setupRegion(m configmap.Mapper) {
 	region, ok := m.Get("region")
-	if !ok {
+	if !ok || region == "" {
 		log.Fatalf("No region set\n")
 	}
 	rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region)

cmd/about/about.go

@@ -44,10 +44,10 @@ var commandDefinition = &cobra.Command{
 	Use:   "about remote:",
 	Short: `Get quota information from the remote.`,
 	Long: `
-` + "`rclone about`" + `prints quota information about a remote to standard
+` + "`rclone about`" + ` prints quota information about a remote to standard
 output. The output is typically used, free, quota and trash contents.
 
-E.g. Typical output from` + "`rclone about remote:`" + `is:
+E.g. Typical output from ` + "`rclone about remote:`" + ` is:
 
     Total:   17G
     Used:    7.444G
@@ -75,7 +75,7 @@ Applying a ` + "`--full`" + ` flag to the command prints the bytes in full, e.g.
     Trashed: 104857602
     Other:   8849156022
 
-A ` + "`--json`" + `flag generates conveniently computer readable output, e.g.
+A ` + "`--json`" + ` flag generates conveniently computer readable output, e.g.
 
     {
         "total": 18253611008,

cmd/cmd.go

@@ -75,8 +75,19 @@ const (
 
 // ShowVersion prints the version to stdout
 func ShowVersion() {
+	osVersion, osKernel := buildinfo.GetOSVersion()
+	if osVersion == "" {
+		osVersion = "unknown"
+	}
+	if osKernel == "" {
+		osKernel = "unknown"
+	}
+
 	linking, tagString := buildinfo.GetLinkingAndTags()
+
 	fmt.Printf("rclone %s\n", fs.Version)
+	fmt.Printf("- os/version: %s\n", osVersion)
+	fmt.Printf("- os/kernel: %s\n", osKernel)
 	fmt.Printf("- os/type: %s\n", runtime.GOOS)
 	fmt.Printf("- os/arch: %s\n", runtime.GOARCH)
 	fmt.Printf("- go/version: %s\n", runtime.Version())
@@ -553,7 +564,7 @@ func Main() {
 	setupRootCommand(Root)
 	AddBackendFlags()
 	if err := Root.Execute(); err != nil {
-		if strings.HasPrefix(err.Error(), "unknown command") {
+		if strings.HasPrefix(err.Error(), "unknown command") && selfupdateEnabled {
 			Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath())
 		}
 		log.Fatalf("Fatal error: %v", err)

cmd/cmount/mount.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rclone/rclone/cmd/mountlib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/buildinfo"
 	"github.com/rclone/rclone/vfs"
 )
 
@@ -35,6 +36,7 @@ func init() {
 		cmd.Aliases = append(cmd.Aliases, "cmount")
 	}
 	mountlib.AddRc("cmount", mount)
+	buildinfo.Tags = append(buildinfo.Tags, "cmount")
 }
 
 // Find the option string in the current options

cmd/copyurl/copyurl.go

@@ -36,7 +36,7 @@ var commandDefinition = &cobra.Command{
 Download a URL's content and copy it to the destination without saving
 it in temporary storage.
 
-Setting ` + "`--auto-filename`" + `will cause the file name to be retrieved from
+Setting ` + "`--auto-filename`" + ` will cause the file name to be retrieved from
 the from URL (after any redirections) and used in the destination
 path. With ` + "`--print-filename`" + ` in addition, the resuling file name will
 be printed.

cmd/link/link.go

@@ -3,7 +3,6 @@ package link
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
@@ -13,7 +12,7 @@ import (
 )
 
 var (
-	expire = fs.Duration(time.Hour * 24 * 365 * 100)
+	expire = fs.DurationOff
 	unlink = false
 )
 

cmd/mountlib/mount.go

@@ -334,7 +334,7 @@ metadata about files like in UNIX. One case that may arise is that other program
 (incorrectly) interprets this as the file being accessible by everyone. For example
 an SSH client may warn about "unprotected private key file".
 
-WinFsp 2021 (version 1.9, still in beta) introduces a new FUSE option "FileSecurity",
+WinFsp 2021 (version 1.9) introduces a new FUSE option "FileSecurity",
 that allows the complete specification of file security descriptors using
 [SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
 With this you can work around issues such as the mentioned "unprotected private key file"

cmd/selfupdate/help.go

@@ -1,3 +1,5 @@
+// +build !noselfupdate
+
 package selfupdate
 
 // Note: "|" will be replaced by backticks in the help string below
@@ -27,7 +29,7 @@ If the old version contains only dots and digits (for example |v1.54.0|)
 then it's a stable release so you won't need the |--beta| flag. Beta releases
 have an additional information similar to |v1.54.0-beta.5111.06f1c0c61|.
 (if you are a developer and use a locally built rclone, the version number
-will end with |-DEV|, you will have to rebuild it as it obvisously can't
+will end with |-DEV|, you will have to rebuild it as it obviously can't
 be distributed).
 
 If you previously installed rclone via a package manager, the package may

cmd/selfupdate/noselfupdate.go

@@ -0,0 +1,11 @@
+// +build noselfupdate
+
+package selfupdate
+
+import (
+	"github.com/rclone/rclone/lib/buildinfo"
+)
+
+func init() {
+	buildinfo.Tags = append(buildinfo.Tags, "noselfupdate")
+}

cmd/selfupdate/selfupdate.go

@@ -1,3 +1,5 @@
+// +build !noselfupdate
+
 package selfupdate
 
 import (
@@ -143,14 +145,9 @@ func InstallUpdate(ctx context.Context, opt *Options) error {
 		return errors.New("--stable and --beta are mutually exclusive")
 	}
 
-	gotCmount := false
-	for _, tag := range buildinfo.Tags {
-		if tag == "cmount" {
-			gotCmount = true
-			break
-		}
-	}
-	if gotCmount && !cmount.ProvidedBy(runtime.GOOS) {
+	// The `cmount` tag is added by cmd/cmount/mount.go only if build is static.
+	_, tags := buildinfo.GetLinkingAndTags()
+	if strings.Contains(" "+tags+" ", " cmount ") && !cmount.ProvidedBy(runtime.GOOS) {
 		return errors.New("updating would discard the mount FUSE capability, aborting")
 	}
 

cmd/selfupdate/selfupdate_test.go

@@ -1,3 +1,5 @@
+// +build !noselfupdate
+
 package selfupdate
 
 import (

cmd/selfupdate/verify.go

@@ -1,3 +1,5 @@
+// +build !noselfupdate
+
 package selfupdate
 
 import (

cmd/selfupdate/writable_unix.go

@@ -1,4 +1,5 @@
 // +build !windows,!plan9,!js
+// +build !noselfupdate
 
 package selfupdate
 

cmd/selfupdate/writable_unsupported.go

@@ -1,4 +1,5 @@
 // +build plan9 js
+// +build !noselfupdate
 
 package selfupdate
 

cmd/selfupdate/writable_windows.go

@@ -1,4 +1,5 @@
 // +build windows
+// +build !noselfupdate
 
 package selfupdate
 

cmd/selfupdate_disabled.go

@@ -0,0 +1,5 @@
+// +build noselfupdate
+
+package cmd
+
+const selfupdateEnabled = false

cmd/selfupdate_enabled.go

@@ -0,0 +1,7 @@
+// +build !noselfupdate
+
+package cmd
+
+// This constant must be in the `cmd` package rather than `cmd/selfupdate`
+// to prevent build failure due to dependency loop.
+const selfupdateEnabled = true

cmd/version/version.go

@@ -29,13 +29,16 @@ var commandDefinition = &cobra.Command{
 	Use:   "version",
 	Short: `Show the version number.`,
 	Long: `
-Show the rclone version number, the go version, the build target OS and
-architecture, build tags and the type of executable (static or dynamic).
+Show the rclone version number, the go version, the build target
+OS and architecture, the runtime OS and kernel version and bitness,
+build tags and the type of executable (static or dynamic).
 
 For example:
 
     $ rclone version
-    rclone v1.54
+    rclone v1.55.0
+    - os/version: ubuntu 18.04 (64 bit)
+    - os/kernel: 4.15.0-136-generic (x86_64)
     - os/type: linux
     - os/arch: amd64
     - go/version: go1.16

docs/content/authors.md

@@ -478,3 +478,4 @@ put them back in again.` >}}
   * Manish Kumar <krmanish260@gmail.com>
   * x0b <x0bdev@gmail.com>
   * CERN through the CS3MESH4EOSC Project
+  * Nick Gaya <nicholasgaya+github@gmail.com>

docs/content/docs.md

@@ -654,7 +654,7 @@ If you run `rclone config file` you will see where the default
 location is for you.
 
 Use this flag to override the config location, e.g. `rclone
---config=".myconfig" .config`.
+--config=".myconfig" config`.
 
 If the location is set to empty string `""` or the special value
 `/notfound`, or the os null device represented by value `NUL` on
@@ -2119,7 +2119,7 @@ mys3:
 Note that if you want to create a remote using environment variables
 you must create the `..._TYPE` variable as above.
 
-Note also that now rclone has [connectionstrings](#connection-strings),
+Note also that now rclone has [connection strings](#connection-strings),
 it is probably easier to use those instead which makes the above example
 
     rclone lsd :s3,access_key_id=XXX,secret_access_key=XXX:

docs/content/dropbox.md

@@ -285,6 +285,12 @@ dropbox:dir` will return the error `Failed to purge: There are too
 many files involved in this operation`.  As a work-around do an
 `rclone delete dropbox:dir` followed by an `rclone rmdir dropbox:dir`.
 
+When using `rclone link` you'll need to set `--expire` if using a
+non-personal account otherwise the visibility may not be correct.
+(Note that `--expire` isn't supported on personal accounts). See the
+[forum discussion](https://forum.rclone.org/t/rclone-link-dropbox-permissions/23211) and the 
+[dropbox SDK issue](https://github.com/dropbox/dropbox-sdk-go-unofficial/issues/75).
+
 ### Get your own Dropbox App ID ###
 
 When you use rclone with Dropbox in its default configuration you are using rclone's App ID. This is shared between all the rclone users.

docs/content/flags.md

@@ -415,6 +415,7 @@ and may be set in the config file.
       --onedrive-link-password string                            Set the password for links created by the link command.
       --onedrive-link-scope string                               Set the scope of the links created by the link command. (default "anonymous")
       --onedrive-link-type string                                Set the type of the links created by the link command. (default "view")
+      --onedrive-list-chunk int                                  Size of listing chunk. (default 1000)
       --onedrive-no-versions                                     Remove all versions on modifying operations
       --onedrive-region string                                   Choose national cloud region for OneDrive. (default "global")
       --onedrive-server-side-across-configs                      Allow server-side operations (e.g. copy) to work across different onedrive configs.

docs/content/install.sh

@@ -101,12 +101,12 @@ case "$OS_type" in
   i?86|x86)
     OS_type='386'
     ;;
+  aarch64|arm64)
+    OS_type='arm64'
+    ;;
   arm*)
     OS_type='arm'
     ;;
-  aarch64)
-    OS_type='arm64'
-    ;;
   *)
     echo 'OS type not supported'
     exit 2

docs/content/onedrive.md

@@ -325,6 +325,15 @@ fall back to normal copy (which will be slightly slower).
 - Type:        bool
 - Default:     false
 
+#### --onedrive-list-chunk
+
+Size of listing chunk.
+
+- Config:      list_chunk
+- Env Var:     RCLONE_ONEDRIVE_LIST_CHUNK
+- Type:        int
+- Default:     1000
+
 #### --onedrive-no-versions
 
 Remove all versions on modifying operations

docs/layouts/partials/version.html

@@ -1 +1 @@
-v1.55.0
+v1.56.0

fs/config/authorize.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
 )
 
 // Authorize is for remote authorization of headless machines.
@@ -16,33 +18,61 @@ import (
 func Authorize(ctx context.Context, args []string, noAutoBrowser bool) error {
 	ctx = suppressConfirm(ctx)
 	switch len(args) {
-	case 1, 3:
+	case 1, 2, 3:
 	default:
 		return errors.Errorf("invalid number of arguments: %d", len(args))
 	}
-	newType := args[0]
-	f := fs.MustFind(newType)
-	if f.Config == nil {
-		return errors.Errorf("can't authorize fs %q", newType)
+	Type := args[0] // FIXME could read this from input
+	ri, err := fs.Find(Type)
+	if err != nil {
+		return err
+	}
+	if ri.Config == nil {
+		return errors.Errorf("can't authorize fs %q", Type)
 	}
-	// Name used for temporary fs
-	name := "**temp-fs**"
 
-	// Make sure we delete it
-	defer DeleteRemote(name)
+	// Config map for remote
+	inM := configmap.Simple{}
 
 	// Indicate that we are running rclone authorize
-	Data.SetValue(name, ConfigAuthorize, "true")
+	inM[ConfigAuthorize] = "true"
 	if noAutoBrowser {
-		Data.SetValue(name, ConfigAuthNoBrowser, "true")
+		inM[ConfigAuthNoBrowser] = "true"
 	}
 
-	if len(args) == 3 {
-		Data.SetValue(name, ConfigClientID, args[1])
-		Data.SetValue(name, ConfigClientSecret, args[2])
+	// Add extra parameters if supplied
+	if len(args) == 2 {
+		err := inM.Decode(args[1])
+		if err != nil {
+			return err
+		}
+	} else if len(args) == 3 {
+		inM[ConfigClientID] = args[1]
+		inM[ConfigClientSecret] = args[2]
 	}
 
-	m := fs.ConfigMap(f, name, nil)
-	f.Config(ctx, name, m)
+	// Name used for temporary remote
+	name := "**temp-fs**"
+
+	m := fs.ConfigMap(ri, name, inM)
+	outM := configmap.Simple{}
+	m.ClearSetters()
+	m.AddSetter(outM)
+	m.AddGetter(outM, configmap.PriorityNormal)
+
+	ri.Config(ctx, name, m)
+
+	// Print the code for the user to paste
+	out := outM["token"]
+
+	// If received a config blob, then return one
+	if len(args) == 2 {
+		out, err = outM.Encode()
+		if err != nil {
+			return err
+		}
+	}
+	fmt.Printf("Paste the following into your remote machine --->\n%s\n<---End paste\n", out)
+
 	return nil
 }

fs/config/configmap/configmap.go

@@ -2,8 +2,24 @@
 package configmap
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"sort"
 	"strings"
+	"unicode"
+
+	"github.com/pkg/errors"
+)
+
+// Priority of getters
+type Priority int8
+
+// Priority levels for AddGetter
+const (
+	PriorityNormal  Priority = iota
+	PriorityConfig           // use for reading from the config
+	PriorityDefault          // use for default values
+	PriorityMax
 )
 
 // Getter provides an interface to get config items
@@ -30,8 +46,12 @@ type Mapper interface {
 // Getter interfaces.
 type Map struct {
 	setters []Setter
-	getters  []Getter
-	override []Getter
+	getters []getprio
+}
+
+type getprio struct {
+	getter   Getter
+	priority Priority
 }
 
 // New returns an empty Map
@@ -39,18 +59,12 @@ func New() *Map {
 	return &Map{}
 }
 
-// AddGetter appends a getter onto the end of the getters
-func (c *Map) AddGetter(getter Getter) *Map {
-	c.getters = append(c.getters, getter)
-	return c
-}
-
-// AddOverrideGetter appends a getter onto the end of the getters
-//
-// It also appends it onto the override getters for GetOverride
-func (c *Map) AddOverrideGetter(getter Getter) *Map {
-	c.getters = append(c.getters, getter)
-	c.override = append(c.override, getter)
+// AddGetter appends a getter onto the end of the getters in priority order
+func (c *Map) AddGetter(getter Getter, priority Priority) *Map {
+	c.getters = append(c.getters, getprio{getter, priority})
+	sort.SliceStable(c.getters, func(i, j int) bool {
+		return c.getters[i].priority < c.getters[j].priority
+	})
 	return c
 }
 
@@ -60,12 +74,34 @@ func (c *Map) AddSetter(setter Setter) *Map {
 	return c
 }
 
-// get gets an item with the key passed in and return the value from
-// the first getter. If the item is found then it returns true,
-// otherwise false.
-func (c *Map) get(key string, getters []Getter) (value string, ok bool) {
-	for _, do := range getters {
-		value, ok = do.Get(key)
+// ClearSetters removes all the setters set so far
+func (c *Map) ClearSetters() *Map {
+	c.setters = nil
+	return c
+}
+
+// ClearGetters removes all the getters with the priority given
+func (c *Map) ClearGetters(priority Priority) *Map {
+	getters := c.getters[:0]
+	for _, item := range c.getters {
+		if item.priority != priority {
+			getters = append(getters, item)
+		}
+	}
+	c.getters = getters
+	return c
+}
+
+// GetPriority gets an item with the key passed in and return the
+// value from the first getter to return a result with priority <=
+// maxPriority. If the item is found then it returns true, otherwise
+// false.
+func (c *Map) GetPriority(key string, maxPriority Priority) (value string, ok bool) {
+	for _, item := range c.getters {
+		if item.priority > maxPriority {
+			break
+		}
+		value, ok = item.getter.Get(key)
 		if ok {
 			return value, ok
 		}
@@ -77,14 +113,7 @@ func (c *Map) get(key string, getters []Getter) (value string, ok bool) {
 // the first getter. If the item is found then it returns true,
 // otherwise false.
 func (c *Map) Get(key string) (value string, ok bool) {
-	return c.get(key, c.getters)
-}
-
-// GetOverride gets an item with the key passed in and return the
-// value from the first override getter. If the item is found then it
-// returns true, otherwise false.
-func (c *Map) GetOverride(key string) (value string, ok bool) {
-	return c.get(key, c.override)
+	return c.GetPriority(key, PriorityMax)
 }
 
 // Set sets an item into all the stored setters.
@@ -135,3 +164,38 @@ func (c Simple) String() string {
 	}
 	return out.String()
 }
+
+// Encode from c into a string suitable for putting on the command line
+func (c Simple) Encode() (string, error) {
+	if len(c) == 0 {
+		return "", nil
+	}
+	buf, err := json.Marshal(c)
+	if err != nil {
+		return "", errors.Wrap(err, "encode simple map")
+	}
+	return base64.RawStdEncoding.EncodeToString(buf), nil
+}
+
+// Decode an Encode~d string in into c
+func (c Simple) Decode(in string) error {
+	// Remove all whitespace from the input string
+	in = strings.Map(func(r rune) rune {
+		if unicode.IsSpace(r) {
+			return -1
+		}
+		return r
+	}, in)
+	if len(in) == 0 {
+		return nil
+	}
+	decodedM, err := base64.RawStdEncoding.DecodeString(in)
+	if err != nil {
+		return errors.Wrap(err, "decode simple map")
+	}
+	err = json.Unmarshal(decodedM, &c)
+	if err != nil {
+		return errors.Wrap(err, "parse simple map")
+	}
+	return nil
+}

fs/config/configmap/configmap_test.go

@@ -1,9 +1,11 @@
 package configmap
 
 import (
+	"encoding/base64"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var (
@@ -27,7 +29,7 @@ func TestConfigMapGet(t *testing.T) {
 		"config1": "one",
 	}
 
-	m.AddGetter(m1)
+	m.AddGetter(m1, PriorityNormal)
 
 	value, found = m.Get("config1")
 	assert.Equal(t, "one", value)
@@ -42,7 +44,7 @@ func TestConfigMapGet(t *testing.T) {
 		"config2": "two2",
 	}
 
-	m.AddGetter(m2)
+	m.AddGetter(m2, PriorityNormal)
 
 	value, found = m.Get("config1")
 	assert.Equal(t, "one", value)
@@ -88,56 +90,160 @@ func TestConfigMapSet(t *testing.T) {
 		"config1": "beetroot",
 		"config2": "potato",
 	}, m2)
+
+	m.ClearSetters()
+
+	// Check that nothing gets set
+	m.Set("config1", "BEETROOT")
+
+	assert.Equal(t, Simple{
+		"config1": "beetroot",
+		"config2": "potato",
+	}, m1)
+	assert.Equal(t, Simple{
+		"config1": "beetroot",
+		"config2": "potato",
+	}, m2)
+
 }
 
-func TestConfigMapGetOverride(t *testing.T) {
+func TestConfigMapGetPriority(t *testing.T) {
 	m := New()
 
-	value, found := m.GetOverride("config1")
+	value, found := m.GetPriority("config1", PriorityMax)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
-	value, found = m.GetOverride("config2")
+	value, found = m.GetPriority("config2", PriorityMax)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
 	m1 := Simple{
 		"config1": "one",
+		"config3": "three",
 	}
 
-	m.AddOverrideGetter(m1)
+	m.AddGetter(m1, PriorityConfig)
 
-	value, found = m.GetOverride("config1")
+	value, found = m.GetPriority("config1", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config2", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config3", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config1", PriorityConfig)
 	assert.Equal(t, "one", value)
 	assert.Equal(t, true, found)
 
-	value, found = m.GetOverride("config2")
+	value, found = m.GetPriority("config2", PriorityConfig)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
+	value, found = m.GetPriority("config3", PriorityConfig)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config1", PriorityMax)
+	assert.Equal(t, "one", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config2", PriorityMax)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config3", PriorityMax)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
 	m2 := Simple{
 		"config1": "one2",
 		"config2": "two2",
 	}
 
-	m.AddGetter(m2)
+	m.AddGetter(m2, PriorityNormal)
 
-	value, found = m.GetOverride("config1")
-	assert.Equal(t, "one", value)
+	value, found = m.GetPriority("config1", PriorityNormal)
+	assert.Equal(t, "one2", value)
 	assert.Equal(t, true, found)
 
-	value, found = m.GetOverride("config2")
-	assert.Equal(t, "", value)
-	assert.Equal(t, false, found)
-
-	value, found = m.Get("config1")
-	assert.Equal(t, "one", value)
-	assert.Equal(t, true, found)
-
-	value, found = m.Get("config2")
+	value, found = m.GetPriority("config2", PriorityNormal)
 	assert.Equal(t, "two2", value)
 	assert.Equal(t, true, found)
 
+	value, found = m.GetPriority("config3", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config1", PriorityConfig)
+	assert.Equal(t, "one2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config2", PriorityConfig)
+	assert.Equal(t, "two2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config3", PriorityConfig)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config1", PriorityMax)
+	assert.Equal(t, "one2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config2", PriorityMax)
+	assert.Equal(t, "two2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config3", PriorityMax)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+}
+
+func TestConfigMapClearGetters(t *testing.T) {
+	m := New()
+	m1 := Simple{}
+	m2 := Simple{}
+	m3 := Simple{}
+	m.AddGetter(m1, PriorityNormal)
+	m.AddGetter(m2, PriorityDefault)
+	m.AddGetter(m3, PriorityConfig)
+	assert.Equal(t, []getprio{
+		{m1, PriorityNormal},
+		{m3, PriorityConfig},
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityConfig)
+	assert.Equal(t, []getprio{
+		{m1, PriorityNormal},
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityNormal)
+	assert.Equal(t, []getprio{
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityDefault)
+	assert.Equal(t, []getprio{}, m.getters)
+	m.ClearGetters(PriorityDefault)
+	assert.Equal(t, []getprio{}, m.getters)
+}
+
+func TestConfigMapClearSetters(t *testing.T) {
+	m := New()
+	m1 := Simple{}
+	m2 := Simple{}
+	m3 := Simple{}
+	m.AddSetter(m1)
+	m.AddSetter(m2)
+	m.AddSetter(m3)
+	assert.Equal(t, []Setter{m1, m2, m3}, m.setters)
+	m.ClearSetters()
+	assert.Equal(t, []Setter(nil), m.setters)
 }
 
 func TestSimpleString(t *testing.T) {
@@ -163,3 +269,91 @@ func TestSimpleString(t *testing.T) {
 		"apple":   "",
 	}.String())
 }
+
+func TestSimpleEncode(t *testing.T) {
+	for _, test := range []struct {
+		in   Simple
+		want string
+	}{
+		{
+			in:   Simple{},
+			want: "",
+		},
+		{
+			in: Simple{
+				"one": "potato",
+			},
+			want: "eyJvbmUiOiJwb3RhdG8ifQ",
+		},
+		{
+			in: Simple{
+				"one": "potato",
+				"two": "",
+			},
+			want: "eyJvbmUiOiJwb3RhdG8iLCJ0d28iOiIifQ",
+		},
+	} {
+		got, err := test.in.Encode()
+		require.NoError(t, err)
+		assert.Equal(t, test.want, got)
+		gotM := Simple{}
+		err = gotM.Decode(got)
+		require.NoError(t, err)
+		assert.Equal(t, test.in, gotM)
+	}
+}
+
+func TestSimpleDecode(t *testing.T) {
+	for _, test := range []struct {
+		in      string
+		want    Simple
+		wantErr string
+	}{
+		{
+			in:   "",
+			want: Simple{},
+		},
+		{
+			in: "eyJvbmUiOiJwb3RhdG8ifQ",
+			want: Simple{
+				"one": "potato",
+			},
+		},
+		{
+			in: "   e yJvbm  UiOiJwb\r\n 3Rhd\tG8ifQ\n\n ",
+			want: Simple{
+				"one": "potato",
+			},
+		},
+		{
+			in: "eyJvbmUiOiJwb3RhdG8iLCJ0d28iOiIifQ",
+			want: Simple{
+				"one": "potato",
+				"two": "",
+			},
+		},
+		{
+			in:      "!!!!!",
+			want:    Simple{},
+			wantErr: "decode simple map",
+		},
+		{
+			in:   base64.RawStdEncoding.EncodeToString([]byte(`null`)),
+			want: Simple{},
+		},
+		{
+			in:      base64.RawStdEncoding.EncodeToString([]byte(`rubbish`)),
+			want:    Simple{},
+			wantErr: "parse simple map",
+		},
+	} {
+		got := Simple{}
+		err := got.Decode(test.in)
+		assert.Equal(t, test.want, got, test.in)
+		if test.wantErr == "" {
+			require.NoError(t, err, test.in)
+		} else {
+			assert.Contains(t, err.Error(), test.wantErr, test.in)
+		}
+	}
+}

fs/fs.go

@@ -125,6 +125,38 @@ func (os Options) Get(name string) *Option {
 	return nil
 }
 
+// Overridden discovers which config items have been overridden in the
+// configmap passed in, either by the config string, command line
+// flags or environment variables
+func (os Options) Overridden(m *configmap.Map) configmap.Simple {
+	var overridden = configmap.Simple{}
+	for i := range os {
+		opt := &os[i]
+		value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal)
+		if isSet {
+			overridden.Set(opt.Name, value)
+		}
+	}
+	return overridden
+}
+
+// NonDefault discovers which config values aren't at their default
+func (os Options) NonDefault(m configmap.Getter) configmap.Simple {
+	var nonDefault = configmap.Simple{}
+	for i := range os {
+		opt := &os[i]
+		value, isSet := m.Get(opt.Name)
+		if !isSet {
+			continue
+		}
+		defaultValue := fmt.Sprint(opt.Default)
+		if value != defaultValue {
+			nonDefault.Set(opt.Name, value)
+		}
+	}
+	return nonDefault
+}
+
 // OptionVisibility controls whether the options are visible in the
 // configurator or the command line.
 type OptionVisibility byte
@@ -1321,28 +1353,28 @@ func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig config
 
 	// Config from connection string
 	if len(connectionStringConfig) > 0 {
-		config.AddOverrideGetter(connectionStringConfig)
+		config.AddGetter(connectionStringConfig, configmap.PriorityNormal)
 	}
 
 	// flag values
 	if fsInfo != nil {
-		config.AddOverrideGetter(&regInfoValues{fsInfo, false})
+		config.AddGetter(&regInfoValues{fsInfo, false}, configmap.PriorityNormal)
 	}
 
 	// remote specific environment vars
-	config.AddOverrideGetter(configEnvVars(configName))
+	config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
 
 	// backend specific environment vars
 	if fsInfo != nil {
-		config.AddOverrideGetter(optionEnvVars{fsInfo: fsInfo})
+		config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal)
 	}
 
 	// config file
-	config.AddGetter(getConfigFile(configName))
+	config.AddGetter(getConfigFile(configName), configmap.PriorityConfig)
 
 	// default values
 	if fsInfo != nil {
-		config.AddGetter(&regInfoValues{fsInfo, true})
+		config.AddGetter(&regInfoValues{fsInfo, true}, configmap.PriorityDefault)
 	}
 
 	// Set Config
@@ -1381,17 +1413,7 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Now discover which config items have been overridden,
-	// either by the config string, command line flags or
-	// environment variables
-	var overridden = configmap.Simple{}
-	for i := range fsInfo.Options {
-		opt := &fsInfo.Options[i]
-		value, isSet := config.GetOverride(opt.Name)
-		if isSet {
-			overridden.Set(opt.Name, value)
-		}
-	}
+	overridden := fsInfo.Options.Overridden(config)
 	if len(overridden) > 0 {
 		extraConfig := overridden.String()
 		//Debugf(nil, "detected overriden config %q", extraConfig)
@@ -1409,6 +1431,11 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
 		// These need to work as filesystem names as the VFS cache will use them
 		configName += suffix
 	}
+	Debugf(nil, "Config dump:")
+	nonDefaultConfig := fsInfo.Options.NonDefault(config)
+	for k, v := range nonDefaultConfig {
+		Debugf(nil, "  %s = %q", k, v)
+	}
 	return fsInfo.NewFs(ctx, configName, fsPath, config)
 }
 

fs/fs_test.go

@@ -190,6 +190,43 @@ func TestOptionsGet(t *testing.T) {
 	assert.Nil(t, opt)
 }
 
+func TestOptionsOveridden(t *testing.T) {
+	m := configmap.New()
+	m1 := configmap.Simple{
+		"nounc":      "m1",
+		"copy_links": "m1",
+	}
+	m.AddGetter(m1, configmap.PriorityNormal)
+	m2 := configmap.Simple{
+		"nounc":            "m2",
+		"case_insensitive": "m2",
+	}
+	m.AddGetter(m2, configmap.PriorityConfig)
+	m3 := configmap.Simple{
+		"nounc": "m3",
+	}
+	m.AddGetter(m3, configmap.PriorityDefault)
+	got := testOptions.Overridden(m)
+	assert.Equal(t, configmap.Simple{
+		"copy_links": "m1",
+		"nounc":      "m1",
+	}, got)
+}
+
+func TestOptionsNonDefault(t *testing.T) {
+	m := configmap.Simple{}
+	got := testOptions.NonDefault(m)
+	assert.Equal(t, configmap.Simple{}, got)
+
+	m["case_insensitive"] = "false"
+	got = testOptions.NonDefault(m)
+	assert.Equal(t, configmap.Simple{}, got)
+
+	m["case_insensitive"] = "true"
+	got = testOptions.NonDefault(m)
+	assert.Equal(t, configmap.Simple{"case_insensitive": "true"}, got)
+}
+
 func TestOptionMarshalJSON(t *testing.T) {
 	out, err := json.MarshalIndent(&caseInsensitiveOption, "", "")
 	assert.NoError(t, err)

fs/sync/sync.go

@@ -155,7 +155,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 	// Input context - cancel this for graceful stop
 	s.inCtx, s.inCancel = context.WithCancel(s.ctx)
 	if s.noTraverse && s.deleteMode != fs.DeleteModeOff {
+		if !fi.HaveFilesFrom() {
 			fs.Errorf(nil, "Ignoring --no-traverse with sync")
+		}
 		s.noTraverse = false
 	}
 	s.trackRenamesStrategy, err = parseTrackRenamesStrategy(ci.TrackRenamesStrategy)
@@ -264,6 +266,9 @@ func (s *syncCopyMove) processError(err error) {
 			// Cancel the march and stop the pipes
 			s.inCancel()
 		}
+	} else if err == context.Canceled && s.inCtx.Err() != nil {
+		// Ignore context Canceled if we have called s.inCancel()
+		return
 	}
 	s.errorMu.Lock()
 	defer s.errorMu.Unlock()

fs/version.go

@@ -1,4 +1,4 @@
 package fs
 
 // Version of rclone
-var Version = "v1.55.0-DEV"
+var Version = "v1.56.0-DEV"

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/nsf/termbox-go v0.0.0-20210114135735-d04385b850e8
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/pkg/sftp v1.13.0
+	github.com/pkg/sftp v1.12.0
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/procfs v0.3.0 // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
@@ -53,6 +53,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sevlyar/go-daemon v0.1.5
+	github.com/shirou/gopsutil/v3 v3.21.3
 	github.com/sirupsen/logrus v1.7.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/cobra v1.1.1

go.sum

@@ -67,6 +67,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/Unknwon/goconfig v0.0.0-20200908083735-df7de6a44db8 h1:1TrMV1HmBApBbM+Hy7RCKZD6UlYWYIPPfoeXomG7+zE=
 github.com/Unknwon/goconfig v0.0.0-20200908083735-df7de6a44db8/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@@ -209,6 +211,8 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -500,6 +504,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/sftp v1.12.0 h1:/f3b24xrDhkhddlaobPe2JgBqfdt+gC/NYl0QY9IOuI=
+github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
 github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI=
 github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -558,6 +564,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk=
 github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE=
+github.com/shirou/gopsutil/v3 v3.21.3 h1:wgcdAHZS2H6qy4JFewVTtqfiYxFzCeEJod/mLztdPG8=
+github.com/shirou/gopsutil/v3 v3.21.3/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@@ -610,6 +618,10 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 h1:IGJQmLBLYBdAknj21W3JsVof0yjEXfy1Q0K3YZebDOg=
 github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
+github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
+github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
@@ -685,6 +697,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -848,6 +861,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 h1:jxr7/dEo+rR29uEBoLSWJ1tRHCFAMwFbGUU9nRqzpds=
 golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

lib/buildinfo/cmount.go

@@ -1,7 +0,0 @@
-// +build cmount
-
-package buildinfo
-
-func init() {
-	Tags = append(Tags, "cmount")
-}

lib/buildinfo/osversion.go

@@ -0,0 +1,34 @@
+// +build !openbsd,!windows
+
+package buildinfo
+
+import (
+	"strings"
+
+	"github.com/shirou/gopsutil/v3/host"
+)
+
+// GetOSVersion returns OS version, kernel and bitness
+func GetOSVersion() (osVersion, osKernel string) {
+	if platform, _, version, err := host.PlatformInformation(); err == nil && platform != "" {
+		osVersion = platform
+		if version != "" {
+			osVersion += " " + version
+		}
+	}
+
+	if version, err := host.KernelVersion(); err == nil && version != "" {
+		osKernel = version
+	}
+
+	if arch, err := host.KernelArch(); err == nil && arch != "" {
+		if strings.HasSuffix(arch, "64") && osVersion != "" {
+			osVersion += " (64 bit)"
+		}
+		if osKernel != "" {
+			osKernel += " (" + arch + ")"
+		}
+	}
+
+	return
+}

lib/buildinfo/osversion_openbsd.go

@@ -0,0 +1,13 @@
+// +build openbsd
+
+package buildinfo
+
+// gopsutil v3.21.3 fails to build on openbsd:
+// Error: .../go/pkg/mod/github.com/tklauser/go-sysconf@v0.3.4/sysconf_openbsd.go:22:28: undefined: unix.RLIMIT_NPROC
+// Error: .../go/pkg/mod/github.com/shirou/gopsutil/v3@v3.21.3/process/process.go:163:15: undefined: pidsWithContext
+// and so on...
+
+// GetOSVersion returns OS version, kernel and bitness
+func GetOSVersion() (osVersion, osKernel string) {
+	return "OpenBSD", ""
+}

lib/buildinfo/osversion_windows.go

@@ -0,0 +1,131 @@
+// +build !openbsd !windows
+
+package buildinfo
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"unsafe"
+
+	"github.com/shirou/gopsutil/v3/host"
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
+)
+
+// GetOSVersion returns OS version, kernel and bitness
+// On Windows it performs additional output enhancements.
+func GetOSVersion() (osVersion, osKernel string) {
+	if platform, _, version, err := host.PlatformInformation(); err == nil && platform != "" {
+		osVersion = platform
+		if version != "" {
+			osVersion += " " + version
+		}
+	}
+
+	if version, err := host.KernelVersion(); err == nil && version != "" {
+		osKernel = version
+
+		// Prevent duplication of output on Windows
+		if strings.Contains(osVersion, osKernel) {
+			deduped := strings.TrimSpace(strings.Replace(osVersion, osKernel, "", 1))
+			if deduped != "" {
+				osVersion = deduped
+			}
+		}
+
+		// Simplify kernel output: `RELEASE.BUILD Build BUILD` -> `RELEASE.BUILD`
+		match := regexp.MustCompile(`^([\d\.]+?\.)(\d+) Build (\d+)$`).FindStringSubmatch(osKernel)
+		if len(match) == 4 && match[2] == match[3] {
+			osKernel = match[1] + match[2]
+		}
+	}
+
+	friendlyName := getRegistryVersionString("ReleaseId")
+	if osVersion != "" && friendlyName != "" {
+		osVersion += " " + friendlyName
+	}
+
+	updateRevision := getRegistryVersionInt("UBR")
+	if osKernel != "" && updateRevision != 0 {
+		osKernel += fmt.Sprintf(".%d", updateRevision)
+	}
+
+	if arch, err := host.KernelArch(); err == nil && arch != "" {
+		if strings.HasSuffix(arch, "64") && osVersion != "" {
+			osVersion += " (64 bit)"
+		}
+		if osKernel != "" {
+			osKernel += " (" + arch + ")"
+		}
+	}
+
+	return
+}
+
+var regVersionKeyUTF16 = windows.StringToUTF16Ptr(`SOFTWARE\Microsoft\Windows NT\CurrentVersion`)
+
+func getRegistryVersionString(name string) string {
+	var (
+		err     error
+		handle  windows.Handle
+		bufLen  uint32
+		valType uint32
+	)
+
+	err = windows.RegOpenKeyEx(windows.HKEY_LOCAL_MACHINE, regVersionKeyUTF16, 0, windows.KEY_READ|windows.KEY_WOW64_64KEY, &handle)
+	if err != nil {
+		return ""
+	}
+	defer func() {
+		_ = windows.RegCloseKey(handle)
+	}()
+
+	nameUTF16 := windows.StringToUTF16Ptr(name)
+	err = windows.RegQueryValueEx(handle, nameUTF16, nil, &valType, nil, &bufLen)
+	if err != nil {
+		return ""
+	}
+
+	regBuf := make([]uint16, bufLen/2+1)
+	err = windows.RegQueryValueEx(handle, nameUTF16, nil, &valType, (*byte)(unsafe.Pointer(&regBuf[0])), &bufLen)
+	if err != nil {
+		return ""
+	}
+
+	return windows.UTF16ToString(regBuf[:])
+}
+
+func getRegistryVersionInt(name string) int {
+	var (
+		err     error
+		handle  windows.Handle
+		bufLen  uint32
+		valType uint32
+	)
+
+	err = windows.RegOpenKeyEx(windows.HKEY_LOCAL_MACHINE, regVersionKeyUTF16, 0, windows.KEY_READ|windows.KEY_WOW64_64KEY, &handle)
+	if err != nil {
+		return 0
+	}
+	defer func() {
+		_ = windows.RegCloseKey(handle)
+	}()
+
+	nameUTF16 := windows.StringToUTF16Ptr(name)
+	err = windows.RegQueryValueEx(handle, nameUTF16, nil, &valType, nil, &bufLen)
+	if err != nil {
+		return 0
+	}
+
+	if valType != registry.DWORD || bufLen != 4 {
+		return 0
+	}
+	var val32 uint32
+	err = windows.RegQueryValueEx(handle, nameUTF16, nil, &valType, (*byte)(unsafe.Pointer(&val32)), &bufLen)
+	if err != nil {
+		return 0
+	}
+
+	return int(val32)
+}

lib/buildinfo/tags.go

@@ -5,7 +5,10 @@ import (
 	"strings"
 )
 
-// Tags contains slice of build tags
+// Tags contains slice of build tags.
+// The `cmount` tag is added by cmd/cmount/mount.go only if build is static.
+// The `noselfupdate` tag is added by cmd/selfupdate/noselfupdate.go
+// Other tags including `cgo` are detected in this package.
 var Tags []string
 
 // GetLinkingAndTags tells how the rclone executable was linked

lib/oauthutil/oauthutil.go

@@ -447,19 +447,46 @@ Execute the following on the machine with the web browser (same rclone
 version recommended):
 
 `)
-			if changed {
-				fmt.Printf("\trclone authorize %q -- %q %q\n", id, oauthConfig.ClientID, oauthConfig.ClientSecret)
+			// Find the configuration
+			ri, err := fs.Find(id)
+			if err != nil {
+				return errors.Wrap(err, "oauthutil authorize")
+			}
+			// Find the overridden options
+			inM := ri.Options.NonDefault(m)
+			delete(inM, config.ConfigToken) // delete token as we are refreshing it
+			for k, v := range inM {
+				fs.Debugf(nil, "sending %s = %q", k, v)
+			}
+			// Encode them into a string
+			mCopyString, err := inM.Encode()
+			if err != nil {
+				return errors.Wrap(err, "oauthutil authorize encode")
+			}
+			// Write what the user has to do
+			if len(mCopyString) > 0 {
+				fmt.Printf("\trclone authorize %q %q\n", id, mCopyString)
 			} else {
 				fmt.Printf("\trclone authorize %q\n", id)
 			}
 			fmt.Println("\nThen paste the result below:")
+			// Read the updates to the config
+			var outM configmap.Simple
+			for {
+				outM = configmap.Simple{}
 				code := config.ReadNonEmptyLine("result> ")
-			token := &oauth2.Token{}
-			err := json.Unmarshal([]byte(code), token)
-			if err != nil {
-				return err
+				err = outM.Decode(code)
+				if err == nil {
+					break
 				}
-			return PutToken(name, m, token, true)
+				fmt.Printf("Couldn't decode response - try again (make sure you are using a matching version of rclone on both sides: %v\n", err)
+			}
+			// Save the config updates
+			for k, v := range outM {
+				m.Set(k, v)
+				fs.Debugf(nil, "received %s = %q", k, v)
+			}
+			return nil
 		}
 	}
 
@@ -526,14 +553,6 @@ version recommended):
 		return errors.Wrap(err, "failed to get token")
 	}
 
-	// Print code if we are doing a manual auth
-	if authorizeOnly {
-		result, err := json.Marshal(token)
-		if err != nil {
-			return errors.Wrap(err, "failed to marshal token")
-		}
-		fmt.Printf("Paste the following into your remote machine --->\n%s\n<---End paste\n", result)
-	}
 	return PutToken(name, m, token, true)
 }