Version v1.52.3

Previous to this change rclone cached the looked up root_folder_id in
the root_folder_id config variable.

This has caused a lot of confusion and a few attempts at workarounds
and ultimately was a mistake.

This reverts rclone attempting to cache anything in root_folder_id and
returns that variable to be entirely user modified.

It gives a little hint in the debug that rclone could be sped up
slightly by setting it, but it is up to the user to think about
whether that would be OK or not.

    Google drive root '': root_folder_id = "XXX" - save this in the config to speed up startup

It does not change root_folder_id itself, leaving this to the user.

See: https://forum.rclone.org/t/rclone-gdrive-no-longer-returning-anything/17215

MANUAL.md

@@ -1,6 +1,6 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Jun 24, 2020
+% Aug 07, 2020
 
 # Rclone syncs your files to cloud storage
 
@@ -330,7 +330,7 @@ kill %1
 
 ## Install from source ##
 
-Make sure you have at least [Go](https://golang.org/) 1.7
+Make sure you have at least [Go](https://golang.org/) 1.10
 installed.  [Download go](https://golang.org/dl/) if necessary.  The
 latest release is recommended. Then
 
@@ -339,16 +339,23 @@ latest release is recommended. Then
     go build
     ./rclone version
 
-You can also build and install rclone in the
+This will leave you a checked out version of rclone you can modify and
-[GOPATH](https://github.com/golang/go/wiki/GOPATH) (which defaults to
+send pull requests with. If you use `make` instead of `go build` then
-`~/go`) with:
+the rclone build will have the correct version information in it.
 
-    go get -u -v github.com/rclone/rclone
+You can also build the latest stable rclone with:
 
-and this will build the binary in `$GOPATH/bin` (`~/go/bin/rclone` by
+    go get github.com/rclone/rclone
-default) after downloading the source to
+
-`$GOPATH/src/github.com/rclone/rclone` (`~/go/src/github.com/rclone/rclone`
+or the latest version (equivalent to the beta) with
-by default).
+
+    go get github.com/rclone/rclone@master
+
+These will build the binary in `$(go env GOPATH)/bin`
+(`~/go/bin/rclone` by default) after downloading the source to the go
+module cache. Note - do **not** use the `-u` flag here. This causes go
+to try to update the depencencies that rclone uses and sometimes these
+don't work with the current version of rclone.
 
 ## Installation with Ansible ##
 
@@ -9683,7 +9690,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format.
       --use-mmap                             Use mmap allocator (see docs).
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
+      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.3")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 
@@ -9893,6 +9900,7 @@ and may be set in the config file.
       --pcloud-client-id string                                  Pcloud App Client Id
       --pcloud-client-secret string                              Pcloud App Client Secret
       --pcloud-encoding MultiEncoder                             This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --pcloud-hostname string                                   Hostname to connect to. (default "api.pcloud.com")
       --pcloud-root-folder-id string                             Fill in for rclone to use a non root folder as its starting point. (default "d0")
       --premiumizeme-encoding MultiEncoder                       This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot)
       --putio-encoding MultiEncoder                              This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
@@ -15337,6 +15345,10 @@ FTP is the File Transfer Protocol. FTP support is provided using the
 [github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
 package.
 
+Paths are specified as `remote:path`. If the path does not begin with
+a `/` it is relative to the home directory of the user.  An empty path
+`remote:` refers to the user's home directory.
+
 Here is an example of making an FTP configuration.  First run
 
     rclone config
@@ -16624,9 +16636,6 @@ Leave blank normally.
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 
-Note that if this is blank, the first time rclone runs it will fill it
-in with the ID of the root folder.
-
 
 - Config:      root_folder_id
 - Env Var:     RCLONE_DRIVE_ROOT_FOLDER_ID
@@ -16833,9 +16842,6 @@ Size of listing chunk 100-1000. 0 to disable.
 
 Impersonate this user when using a service account.
 
-Note that if this is used then "root_folder_id" will be ignored.
-
-
 - Config:      impersonate
 - Env Var:     RCLONE_DRIVE_IMPERSONATE
 - Type:        string
@@ -17232,6 +17238,13 @@ the remote configuration, it's not such a big deal).
 
 (Thanks to @balazer on github for these instructions.)
 
+Sometimes, creation of an OAuth consent in Google API Console fails due to an error message
+“The request failed because changes to one of the field of the resource is not supported”.
+As a convenient workaround, the necessary Google Drive API key can be created on the
+[Python Quickstart](https://developers.google.com/drive/api/v3/quickstart/python) page.
+Just push the Enable the Drive API button to receive the Client ID and Secret.
+Note that it will automatically create a new project in the API Console.
+
  Google Photos
 -------------------------------------------------
 
@@ -19449,7 +19462,7 @@ requests.
 If you are having problems with them (E.g., seeing a lot of throttling), you can get your own
 Client ID and Key by following the steps below:
 
-1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade, then click `New registration`.
+1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI` Enter `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Copy and keep that secret for later use.
 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.
@@ -20869,6 +20882,17 @@ Fill in for rclone to use a non root folder as its starting point.
 - Type:        string
 - Default:     "d0"
 
+#### --pcloud-hostname
+
+Hostname to connect to.
+
+This is normally set when rclone initially does the oauth connection.
+
+- Config:      hostname
+- Env Var:     RCLONE_PCLOUD_HOSTNAME
+- Type:        string
+- Default:     "api.pcloud.com"
+
 
 
  premiumize.me
@@ -23638,6 +23662,36 @@ Options:
 
 # Changelog
 
+## v1.52.3 - 2020-08-07
+
+[See commits](https://github.com/rclone/rclone/compare/v1.52.2...v1.52.3)
+
+* Bug Fixes
+    * docs
+        * Disable smart typography (eg en-dash) in MANUAL.* and man page (Nick Craig-Wood)
+        * Update install.md to reflect minimum Go version (Evan Harris)
+        * Update install from source instructions (Nick Craig-Wood)
+        * make_manual: Support SOURCE_DATE_EPOCH (Morten Linderud)
+    * log: Fix --use-json-log going to stderr not --log-file on Windows (Nick Craig-Wood)
+    * serve dlna: Fix file list on Samsung Series 6+ TVs (Matteo Pietro Dazzi)
+    * sync: Fix deadlock with --track-renames-strategy modtime (Nick Craig-Wood)
+* Cache
+    * Fix moveto/copyto remote:file remote:file2 (Nick Craig-Wood)
+* Drive
+    * Stop using root_folder_id as a cache (Nick Craig-Wood)
+    * Make dangling shortcuts appear in listings (Nick Craig-Wood)
+    * Drop "Disabling ListR" messages down to debug (Nick Craig-Wood)
+    * Workaround and policy for Google Drive API (Dmitry Ustalov)
+* FTP
+    * Add note to docs about home vs root directory selection (Nick Craig-Wood)
+* Onedrive
+    * Fix reverting to Copy when Move would have worked (Nick Craig-Wood)
+    * Avoid comma rendered in URL in onedrive.md (Kevin)
+* Pcloud
+    * Fix oauth on European region "eapi.pcloud.com" (Nick Craig-Wood)
+* S3
+    * Fix bucket Region auto detection when Region unset in config (Nick Craig-Wood)
+
 ## v1.52.2 - 2020-06-24
 
 [See commits](https://github.com/rclone/rclone/compare/v1.52.1...v1.52.2)

Makefile

@@ -105,16 +105,16 @@ tidy:
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
 MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/

VERSION

@@ -1 +1 @@
-v1.52.2
+v1.52.3

backend/drive/drive.go

@@ -57,6 +57,7 @@ const (
 	rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
 	driveFolderType             = "application/vnd.google-apps.folder"
 	shortcutMimeType            = "application/vnd.google-apps.shortcut"
+	shortcutMimeTypeDangling    = "application/vnd.google-apps.shortcut.dangling" // synthetic mime type for internal use
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
@@ -223,9 +224,6 @@ Leave blank normally.
 
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
-
-Note that if this is blank, the first time rclone runs it will fill it
-in with the ID of the root folder.
 `,
 		}, {
 			Name: "service_account_file",
@@ -349,12 +347,9 @@ date is used.`,
 			Help:     "Size of listing chunk 100-1000. 0 to disable.",
 			Advanced: true,
 		}, {
-			Name:    "impersonate",
+			Name:     "impersonate",
-			Default: "",
+			Default:  "",
-			Help: `Impersonate this user when using a service account.
+			Help:     `Impersonate this user when using a service account.`,
-
-Note that if this is used then "root_folder_id" will be ignored.
-`,
 			Advanced: true,
 		}, {
 			Name:    "alternate_export",
@@ -1121,25 +1116,15 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		}
 	}
 
-	// If impersonating warn about root_folder_id if set and unset it
+	// Set the root folder ID
-	//
-	// This is because rclone v1.51 and v1.52 cached root_folder_id when
-	// using impersonate which they shouldn't have done. It is possible
-	// someone is using impersonate and root_folder_id in which case this
-	// breaks their workflow. There isn't an easy way around that.
-	if opt.RootFolderID != "" && opt.Impersonate != "" {
-		fs.Logf(f, "Ignoring cached root_folder_id when using --drive-impersonate")
-		opt.RootFolderID = ""
-	}
-
-	// set root folder for a team drive or query the user root folder
 	if opt.RootFolderID != "" {
-		// override root folder if set or cached in the config and not impersonating
+		// use root_folder ID if set
 		f.rootFolderID = opt.RootFolderID
 	} else if f.isTeamDrive {
+		// otherwise use team_drive if set
 		f.rootFolderID = f.opt.TeamDriveID
 	} else {
-		// Look up the root ID and cache it in the config
+		// otherwise look up the actual root ID
 		rootID, err := f.getRootID()
 		if err != nil {
 			if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
@@ -1151,10 +1136,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 			}
 		}
 		f.rootFolderID = rootID
-		// Don't cache the root folder ID if impersonating
+		fs.Debugf(f, "root_folder_id = %q - save this in the config to speed up startup", rootID)
-		if opt.Impersonate == "" {
-			m.Set("root_folder_id", rootID)
-		}
 	}
 
 	f.dirCache = dircache.New(root, f.rootFolderID, f)
@@ -1356,6 +1338,10 @@ func (f *Fs) newObjectWithExportInfo(
 		// and not from a listing. This is unlikely.
 		fs.Debugf(remote, "Ignoring shortcut as skip shortcuts is set")
 		return nil, fs.ErrorObjectNotFound
+	case info.MimeType == shortcutMimeTypeDangling:
+		// Pretend a dangling shortcut is a regular object
+		// It will error if used, but appear in listings so it can be deleted
+		return f.newRegularObject(remote, info), nil
 	case info.Md5Checksum != "" || info.Size > 0:
 		// If item has MD5 sum or a length it is a file stored on drive
 		return f.newRegularObject(remote, info), nil
@@ -1740,7 +1726,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// https://issuetracker.google.com/issues/149522397
 		if len(dirs) > 1 && !foundItems {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
-				fs.Logf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
+				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
 			var recycled = make([]listREntry, len(dirs))
 			f.listRmu.Lock()
@@ -1773,7 +1759,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 				// empty so must have made a mistake
 				if len(f.listRempties) == 0 {
 					if atomic.SwapInt32(&f.grouping, listRGrouping) != listRGrouping {
-						fs.Logf(f, "Re-enabling ListR as previous detection was in error")
+						fs.Debugf(f, "Re-enabling ListR as previous detection was in error")
 					}
 				}
 			}
@@ -1988,6 +1974,12 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 	}
 	newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
+		if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
+			// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
+			fs.Logf(nil, "Dangling shortcut %q detected", item.Name)
+			item.MimeType = shortcutMimeTypeDangling
+			return item, nil
+		}
 		return nil, errors.Wrap(err, "failed to resolve shortcut")
 	}
 	// make sure we use the Name, Parents and Trashed from the original item
@@ -3296,6 +3288,9 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.mimeType == shortcutMimeTypeDangling {
+		return nil, errors.New("can't read dangling shortcut")
+	}
 	if o.v2Download {
 		var v2File *drive_v2.File
 		err = o.fs.pacer.Call(func() (bool, error) {

backend/onedrive/onedrive.go

@@ -1121,9 +1121,10 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	id, dstDriveID, _ := parseNormalizedID(directoryID)
 	_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
 
-	if dstDriveID != srcObjDriveID {
+	if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
+		fs.Debugf(f, "Can't move files between drives (%q != %q)", dstDriveID, srcObjDriveID)
 		return nil, fs.ErrorCantMove
 	}
 
@@ -1219,9 +1220,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
-	if dstDriveID != srcDriveID {
+	if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
+		fs.Debugf(f, "Can't move directories between drives (%q != %q)", dstDriveID, srcDriveID)
 		return fs.ErrorCantDirMove
 	}
 
@@ -1869,6 +1871,17 @@ func parseNormalizedID(ID string) (string, string, string) {
 	return ID, "", ""
 }
 
+// Returns the canonical form of the driveID
+func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
+	if driveID == "" {
+		canonicalDriveID = f.opt.DriveID
+	} else {
+		canonicalDriveID = driveID
+	}
+	canonicalDriveID = strings.ToLower(canonicalDriveID)
+	return canonicalDriveID
+}
+
 // getRelativePathInsideBase checks if `target` is inside `base`. If so, it
 // returns a relative path for `target` based on `base` and a boolean `true`.
 // Otherwise returns "", false.

backend/pcloud/pcloud.go

@@ -42,7 +42,7 @@ const (
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
-	rootURL                     = "https://api.pcloud.com"
+	defaultHostname             = "api.pcloud.com"
 )
 
 // Globals
@@ -51,8 +51,8 @@ var (
 	oauthConfig = &oauth2.Config{
 		Scopes: nil,
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://my.pcloud.com/oauth2/authorize",
+			AuthURL: "https://my.pcloud.com/oauth2/authorize",
-			TokenURL: "https://api.pcloud.com/oauth2_token",
+			// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
 		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
@@ -60,17 +60,45 @@ var (
 	}
 )
 
+// Update the TokenURL with the actual hostname
+func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
+	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
+}
+
 // Register with Fs
 func init() {
+	updateTokenURL(oauthConfig, defaultHostname)
 	fs.Register(&fs.RegInfo{
 		Name:        "pcloud",
 		Description: "Pcloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
+			optc := new(Options)
+			err := configstruct.Set(m, optc)
+			if err != nil {
+				fs.Errorf(nil, "Failed to read config: %v", err)
+			}
+			updateTokenURL(oauthConfig, optc.Hostname)
+			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
+				if auth == nil || auth.Form == nil {
+					return errors.New("form not found in response")
+				}
+				hostname := auth.Form.Get("hostname")
+				if hostname == "" {
+					hostname = defaultHostname
+				}
+				// Save the hostname in the config
+				m.Set("hostname", hostname)
+				// Update the token URL
+				updateTokenURL(oauthConfig, hostname)
+				fs.Debugf(nil, "pcloud: got hostname %q", hostname)
+				return nil
+			}
 			opt := oauthutil.Options{
+				CheckAuth:    checkAuth,
 				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
 			}
-			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
+			err = oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -96,6 +124,13 @@ func init() {
 			Help:     "Fill in for rclone to use a non root folder as its starting point.",
 			Default:  "d0",
 			Advanced: true,
+		}, {
+			Name: "hostname",
+			Help: `Hostname to connect to.
+
+This is normally set when rclone initially does the oauth connection.`,
+			Default:  defaultHostname,
+			Advanced: true,
 		}},
 	})
 }
@@ -104,6 +139,7 @@ func init() {
 type Options struct {
 	Enc          encoder.MultiEncoder `config:"encoding"`
 	RootFolderID string               `config:"root_folder_id"`
+	Hostname     string               `config:"hostname"`
 }
 
 // Fs represents a remote pcloud
@@ -253,12 +289,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure Pcloud")
 	}
+	updateTokenURL(oauthConfig, opt.Hostname)
 
 	f := &Fs{
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
+		srv:   rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),
 		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{

backend/s3/s3.go

@@ -1127,9 +1127,6 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		return nil, nil, errors.New("secret_access_key not found")
 	}
 
-	if opt.Region == "" && opt.Endpoint == "" {
-		opt.Endpoint = "https://s3.amazonaws.com/"
-	}
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}

bin/make_manual.py

@@ -6,6 +6,7 @@ conversion into man pages etc.
 
 import os
 import re
+import time
 from datetime import datetime
 
 docpath = "docs/content"
@@ -160,13 +161,15 @@ def read_commands(docpath):
 def main():
     check_docs(docpath)
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
+    build_date = datetime.utcfromtimestamp(
+            int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-""" % datetime.now().strftime("%b %d, %Y"))
+""" % build_date.strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

cmd/serve/dlna/cds.go

@@ -77,10 +77,12 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
 	}
 
 	if fileInfo.IsDir() {
+		defaultChildCount := 1
 		obj.Class = "object.container.storageFolder"
 		obj.Title = fileInfo.Name()
 		return upnpav.Container{
-			Object: obj,
+			Object:     obj,
+			ChildCount: &defaultChildCount,
 		}, nil
 	}
 

docs/content/changelog.md

@@ -5,6 +5,36 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.52.3 - 2020-08-07
+
+[See commits](https://github.com/rclone/rclone/compare/v1.52.2...v1.52.3)
+
+* Bug Fixes
+    * docs
+        * Disable smart typography (eg en-dash) in MANUAL.* and man page (Nick Craig-Wood)
+        * Update install.md to reflect minimum Go version (Evan Harris)
+        * Update install from source instructions (Nick Craig-Wood)
+        * make_manual: Support SOURCE_DATE_EPOCH (Morten Linderud)
+    * log: Fix --use-json-log going to stderr not --log-file on Windows (Nick Craig-Wood)
+    * serve dlna: Fix file list on Samsung Series 6+ TVs (Matteo Pietro Dazzi)
+    * sync: Fix deadlock with --track-renames-strategy modtime (Nick Craig-Wood)
+* Cache
+    * Fix moveto/copyto remote:file remote:file2 (Nick Craig-Wood)
+* Drive
+    * Stop using root_folder_id as a cache (Nick Craig-Wood)
+    * Make dangling shortcuts appear in listings (Nick Craig-Wood)
+    * Drop "Disabling ListR" messages down to debug (Nick Craig-Wood)
+    * Workaround and policy for Google Drive API (Dmitry Ustalov)
+* FTP
+    * Add note to docs about home vs root directory selection (Nick Craig-Wood)
+* Onedrive
+    * Fix reverting to Copy when Move would have worked (Nick Craig-Wood)
+    * Avoid comma rendered in URL in onedrive.md (Kevin)
+* Pcloud
+    * Fix oauth on European region "eapi.pcloud.com" (Nick Craig-Wood)
+* S3
+    * Fix bucket Region auto detection when Region unset in config (Nick Craig-Wood)
+
 ## v1.52.2 - 2020-06-24
 
 [See commits](https://github.com/rclone/rclone/compare/v1.52.1...v1.52.2)

docs/content/drive.md

@@ -593,9 +593,6 @@ Leave blank normally.
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 
-Note that if this is blank, the first time rclone runs it will fill it
-in with the ID of the root folder.
-
 
 - Config:      root_folder_id
 - Env Var:     RCLONE_DRIVE_ROOT_FOLDER_ID
@@ -802,9 +799,6 @@ Size of listing chunk 100-1000. 0 to disable.
 
 Impersonate this user when using a service account.
 
-Note that if this is used then "root_folder_id" will be ignored.
-
-
 - Config:      impersonate
 - Env Var:     RCLONE_DRIVE_IMPERSONATE
 - Type:        string
@@ -1200,3 +1194,10 @@ for rclone to be able to get its token-id (but as this only happens during
 the remote configuration, it's not such a big deal). 
 
 (Thanks to @balazer on github for these instructions.)
+
+Sometimes, creation of an OAuth consent in Google API Console fails due to an error message
+“The request failed because changes to one of the field of the resource is not supported”.
+As a convenient workaround, the necessary Google Drive API key can be created on the
+[Python Quickstart](https://developers.google.com/drive/api/v3/quickstart/python) page.
+Just push the Enable the Drive API button to receive the Client ID and Secret.
+Note that it will automatically create a new project in the API Console.

docs/content/flags.md

@@ -144,7 +144,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format.
       --use-mmap                             Use mmap allocator (see docs).
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
+      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.3")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 
@@ -354,6 +354,7 @@ and may be set in the config file.
       --pcloud-client-id string                                  Pcloud App Client Id
       --pcloud-client-secret string                              Pcloud App Client Secret
       --pcloud-encoding MultiEncoder                             This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --pcloud-hostname string                                   Hostname to connect to. (default "api.pcloud.com")
       --pcloud-root-folder-id string                             Fill in for rclone to use a non root folder as its starting point. (default "d0")
       --premiumizeme-encoding MultiEncoder                       This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot)
       --putio-encoding MultiEncoder                              This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)

docs/content/ftp.md

@@ -10,6 +10,10 @@ FTP is the File Transfer Protocol. FTP support is provided using the
 [github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
 package.
 
+Paths are specified as `remote:path`. If the path does not begin with
+a `/` it is relative to the home directory of the user.  An empty path
+`remote:` refers to the user's home directory.
+
 Here is an example of making an FTP configuration.  First run
 
     rclone config

docs/content/install.md

@@ -174,7 +174,7 @@ kill %1
 
 ## Install from source ##
 
-Make sure you have at least [Go](https://golang.org/) 1.7
+Make sure you have at least [Go](https://golang.org/) 1.10
 installed.  [Download go](https://golang.org/dl/) if necessary.  The
 latest release is recommended. Then
 
@@ -183,16 +183,23 @@ latest release is recommended. Then
     go build
     ./rclone version
 
-You can also build and install rclone in the
+This will leave you a checked out version of rclone you can modify and
-[GOPATH](https://github.com/golang/go/wiki/GOPATH) (which defaults to
+send pull requests with. If you use `make` instead of `go build` then
-`~/go`) with:
+the rclone build will have the correct version information in it.
 
-    go get -u -v github.com/rclone/rclone
+You can also build the latest stable rclone with:
 
-and this will build the binary in `$GOPATH/bin` (`~/go/bin/rclone` by
+    go get github.com/rclone/rclone
-default) after downloading the source to
+
-`$GOPATH/src/github.com/rclone/rclone` (`~/go/src/github.com/rclone/rclone`
+or the latest version (equivalent to the beta) with
-by default).
+
+    go get github.com/rclone/rclone@master
+
+These will build the binary in `$(go env GOPATH)/bin`
+(`~/go/bin/rclone` by default) after downloading the source to the go
+module cache. Note - do **not** use the `-u` flag here. This causes go
+to try to update the depencencies that rclone uses and sometimes these
+don't work with the current version of rclone.
 
 ## Installation with Ansible ##
 

docs/content/onedrive.md

@@ -127,7 +127,7 @@ requests.
 If you are having problems with them (E.g., seeing a lot of throttling), you can get your own
 Client ID and Key by following the steps below:
 
-1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade, then click `New registration`.
+1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI` Enter `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Copy and keep that secret for later use.
 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.

docs/content/pcloud.md

@@ -182,4 +182,15 @@ Fill in for rclone to use a non root folder as its starting point.
 - Type:        string
 - Default:     "d0"
 
+#### --pcloud-hostname
+
+Hostname to connect to.
+
+This is normally set when rclone initially does the oauth connection.
+
+- Config:      hostname
+- Env Var:     RCLONE_PCLOUD_HOSTNAME
+- Type:        string
+- Default:     "api.pcloud.com"
+
 {{< rem autogenerated options stop >}}

docs/content/privacy.md

@@ -53,6 +53,14 @@ Users are advised to use social media platforms wisely and communicate / engage
 
 This website may use social sharing buttons which help share web content directly from web pages to the social media platform in question. Users are advised before using such social sharing buttons that they do so at their own discretion and note that the social media platform may track and save your request to share a web page respectively through your social media platform account.
 
+## Use of Cloud API User Data ##
+
+Rclone is a command line program to manage files on cloud storage. Its sole purpose is to access and manipulate user content in the [supported](/overview/) cloud storage systems from a local machine of the end user. For accessing the user content via the cloud provider API, Rclone uses authentication mechanisms, such as OAuth or HTTP Cookies, depending on the particular cloud provider offerings. Use of these authentication mechanisms and user data is governed by the privacy policies mentioned in the [Resources & Further Information](/privacy/#resources-further-information) section and followed by the privacy policy of Rclone.
+
+* Rclone provides the end user with access to their files available in a storage system associated by the authentication credentials via the publicly exposed API of the storage system.
+* Rclone allows storing the authentication credentials on the user machine in the local configuration file.
+* Rclone does not share any user data with third parties.
+
 ## Resources & Further Information ##
 
   * [Data Protection Act 1998](http://www.legislation.gov.uk/ukpga/1998/29/contents)
@@ -61,4 +69,5 @@ This website may use social sharing buttons which help share web content directl
   * [Twitter Privacy Policy](https://twitter.com/privacy)
   * [Facebook Privacy Policy](https://www.facebook.com/about/privacy/)
   * [Google Privacy Policy](https://www.google.com/privacy.html)
+  * [Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy)
   * [Sample Website Privacy Policy](http://www.jamieking.co.uk/resources/free_sample_privacy_policy.html)

docs/layouts/partials/version.html

@@ -1 +1 @@
-v1.52.2
+v1.52.3

fs/cache/cache.go

@@ -56,12 +56,20 @@ func GetFn(fsString string, create func(fsString string) (fs.Fs, error)) (f fs.F
 	if created {
 		canonicalName := fs.ConfigString(f)
 		if canonicalName != fsString {
-			fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", fsString, canonicalName)
+			// Note that if err == fs.ErrorIsFile at this moment
-			value, found := c.Rename(fsString, canonicalName)
+			// then we can't rename the remote as it will have the
-			if found {
+			// wrong error status, we need to add a new one.
-				f = value.(fs.Fs)
+			if err == nil {
+				fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", fsString, canonicalName)
+				value, found := c.Rename(fsString, canonicalName)
+				if found {
+					f = value.(fs.Fs)
+				}
+				addMapping(fsString, canonicalName)
+			} else {
+				fs.Debugf(nil, "fs cache: adding new entry for parent of %q, %q", fsString, canonicalName)
+				Put(canonicalName, f)
 			}
-			addMapping(fsString, canonicalName)
 		}
 	}
 	return f, err

fs/cache/cache_test.go

@@ -23,7 +23,7 @@ func mockNewFs(t *testing.T) (func(), func(path string) (fs.Fs, error)) {
 		switch path {
 		case "mock:/":
 			return mockfs.NewFs("mock", "/"), nil
-		case "mock:/file.txt":
+		case "mock:/file.txt", "mock:file.txt":
 			return mockfs.NewFs("mock", "/"), fs.ErrorIsFile
 		case "mock:/error":
 			return nil, errSentinel
@@ -64,13 +64,46 @@ func TestGetFile(t *testing.T) {
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 2, c.Entries())
 
 	f2, err := GetFn("mock:/file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f2)
 
 	assert.Equal(t, f, f2)
+
+	// check parent is there too
+	f2, err = GetFn("mock:/", create)
+	require.Nil(t, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
+}
+
+func TestGetFile2(t *testing.T) {
+	cleanup, create := mockNewFs(t)
+	defer cleanup()
+
+	assert.Equal(t, 0, c.Entries())
+
+	f, err := GetFn("mock:file.txt", create)
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.NotNil(t, f)
+
+	assert.Equal(t, 2, c.Entries())
+
+	f2, err := GetFn("mock:file.txt", create)
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
+
+	// check parent is there too
+	f2, err = GetFn("mock:/", create)
+	require.Nil(t, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
 }
 
 func TestGetError(t *testing.T) {

fs/log/log.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/sirupsen/logrus"
 )
 
 // Options contains options for the remote control server
@@ -120,6 +121,7 @@ func InitLogging() {
 			fs.Errorf(nil, "Failed to seek log file to end: %v", err)
 		}
 		log.SetOutput(f)
+		logrus.SetOutput(f)
 		redirectStderr(f)
 	}
 

fs/sync/sync.go

@@ -658,6 +658,7 @@ func (s *syncCopyMove) pushRenameMap(hash string, obj fs.Object) {
 // renameMap or returns nil if not found.
 func (s *syncCopyMove) popRenameMap(hash string, src fs.Object) (dst fs.Object) {
 	s.renameMapMu.Lock()
+	defer s.renameMapMu.Unlock()
 	dsts, ok := s.renameMap[hash]
 	if ok && len(dsts) > 0 {
 		// Element to remove
@@ -690,7 +691,6 @@ func (s *syncCopyMove) popRenameMap(hash string, src fs.Object) (dst fs.Object)
 			delete(s.renameMap, hash)
 		}
 	}
-	s.renameMapMu.Unlock()
 	return dst
 }
 

fs/version.go

@@ -1,4 +1,4 @@
 package fs
 
 // Version of rclone
-var Version = "v1.52.2"
+var Version = "v1.52.3"