jottacloud: fix token refresh failed: is not a regular file error

Before this change the jottacloud token renewer would run and give the
error:

    Token refresh failed: is not a regular file

This is because the refresh runs on the root and it isn't a file.

This was fixed by ignoring that specific error.

See: https://forum.rclone.org/t/jottacloud-crypt-3-gb-copy-runs-for-a-week-without-completing/21173

.github/workflows/build.yml

@@ -46,6 +46,7 @@ jobs:
             go: '1.15.x'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
+            build_args: '-buildmode exe'
             quicktest: true
             racequicktest: true
             deploy: true
@@ -57,6 +58,7 @@ jobs:
             goarch: '386'
             cgo: '1'
             build_flags: '-include "^windows/386" -cgo'
+            build_args: '-buildmode exe'
             quicktest: true
             deploy: true
 
@@ -109,6 +111,7 @@ jobs:
         run: |
           echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
           echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
+          echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV
           if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi
           if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
 
@@ -124,10 +127,8 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
-          brew untap local/homebrew-openssl  # workaround for https://github.com/actions/virtual-environments/issues/1811
-          brew untap local/homebrew-python2  # workaround for https://github.com/actions/virtual-environments/issues/1811
           brew update
-          brew cask install osxfuse
+          brew install --cask osxfuse
         if: matrix.os == 'macOS-latest'
 
       - name: Install Libraries on Windows

.github/workflows/build_publish_docker_image.yml

@@ -15,7 +15,7 @@ jobs:
               with:
                 fetch-depth: 0
             - name: Build and publish image
-              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              uses: ilteoood/docker_buildx@1.1.0
               with:
                 tag: beta
                 imageName: rclone/rclone

.github/workflows/build_publish_release_docker_image.yml

@@ -23,7 +23,7 @@ jobs:
               id: actual_major_version
               run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
             - name: Build and publish image
-              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              uses: ilteoood/docker_buildx@1.1.0
               with:
                 tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
                 imageName: rclone/rclone

MANUAL.html

@@ -17,7 +17,7 @@
 <header id="title-block-header">
 <h1 class="title">rclone(1) User Manual</h1>
 <p class="author">Nick Craig-Wood</p>
-<p class="date">Oct 26, 2020</p>
+<p class="date">Nov 19, 2020</p>
 </header>
 <h1 id="rclone-syncs-your-files-to-cloud-storage">Rclone syncs your files to cloud storage</h1>
 <p><img width="50%" src="https://rclone.org/img/logo_on_light__horizontal_color.svg" alt="rclone logo" style="float:right; padding: 5px;" ></p>
@@ -6291,7 +6291,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
       --use-json-log                         Use json log format.
       --use-mmap                             Use mmap allocator (see docs).
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default &quot;rclone/v1.53.2&quot;)
+      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default &quot;rclone/v1.53.3&quot;)
   -v, --verbose count                        Print lots more stuff (repeat for more)</code></pre>
 <h2 id="backend-flags">Backend Flags</h2>
 <p>These flags are available for every command. They control the backends and may be set in the config file.</p>
@@ -18552,6 +18552,27 @@ $ tree /tmp/b
 <li>"error": return an error based on option value</li>
 </ul>
 <h1 id="changelog">Changelog</h1>
+<h2 id="v1.53.3---2020-11-19">v1.53.3 - 2020-11-19</h2>
+<p><a href="https://github.com/rclone/rclone/compare/v1.53.2...v1.53.3">See commits</a></p>
+<ul>
+<li>Bug Fixes
+<ul>
+<li>random: Fix incorrect use of math/rand instead of crypto/rand CVE-2020-28924 (Nick Craig-Wood)
+<ul>
+<li>Passwords you have generated with <code>rclone config</code> may be insecure</li>
+<li>See <a href="https://github.com/rclone/rclone/issues/4783">issue #4783</a> for more details and a checking tool</li>
+</ul></li>
+<li>random: Seed math/rand in one place with crypto strong seed (Nick Craig-Wood)</li>
+</ul></li>
+<li>VFS
+<ul>
+<li>Fix vfs/refresh calls with fs= parameter (Nick Craig-Wood)</li>
+</ul></li>
+<li>Sharefile
+<ul>
+<li>Fix backend due to API swapping integers for strings (Nick Craig-Wood)</li>
+</ul></li>
+</ul>
 <h2 id="v1.53.2---2020-10-26">v1.53.2 - 2020-10-26</h2>
 <p><a href="https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2">See commits</a></p>
 <ul>

MANUAL.md

@@ -1,6 +1,6 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Oct 26, 2020
+% Nov 19, 2020
 
 # Rclone syncs your files to cloud storage
 
@@ -10569,7 +10569,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format.
       --use-mmap                             Use mmap allocator (see docs).
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.2")
+      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.3")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 
@@ -25727,6 +25727,20 @@ Options:
 
 # Changelog
 
+## v1.53.3 - 2020-11-19
+
+[See commits](https://github.com/rclone/rclone/compare/v1.53.2...v1.53.3)
+
+* Bug Fixes
+    * random: Fix incorrect use of math/rand instead of crypto/rand CVE-2020-28924 (Nick Craig-Wood)
+        * Passwords you have generated with `rclone config` may be insecure
+        * See [issue #4783](https://github.com/rclone/rclone/issues/4783) for more details and a checking tool
+    * random: Seed math/rand in one place with crypto strong seed (Nick Craig-Wood)
+* VFS
+    * Fix vfs/refresh calls with fs= parameter (Nick Craig-Wood)
+* Sharefile
+    * Fix backend due to API swapping integers for strings (Nick Craig-Wood)
+
 ## v1.53.2 - 2020-10-26
 
 [See commits](https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)

MANUAL.txt

@@ -1,6 +1,6 @@
 rclone(1) User Manual
 Nick Craig-Wood
-Oct 26, 2020
+Nov 19, 2020
 
 
 
@@ -10660,7 +10660,7 @@ These flags are available for every command.
           --use-json-log                         Use json log format.
           --use-mmap                             Use mmap allocator (see docs).
           --use-server-modtime                   Use server modified time instead of object metadata
-          --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.2")
+          --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.3")
       -v, --verbose count                        Print lots more stuff (repeat for more)
 
 
@@ -25700,6 +25700,25 @@ Options:
 CHANGELOG
 
 
+v1.53.3 - 2020-11-19
+
+See commits
+
+-   Bug Fixes
+    -   random: Fix incorrect use of math/rand instead of crypto/rand
+        CVE-2020-28924 (Nick Craig-Wood)
+        -   Passwords you have generated with rclone config may be
+            insecure
+        -   See issue #4783 for more details and a checking tool
+    -   random: Seed math/rand in one place with crypto strong seed
+        (Nick Craig-Wood)
+-   VFS
+    -   Fix vfs/refresh calls with fs= parameter (Nick Craig-Wood)
+-   Sharefile
+    -   Fix backend due to API swapping integers for strings (Nick
+        Craig-Wood)
+
+
 v1.53.2 - 2020-10-26
 
 See commits

Makefile

@@ -46,13 +46,13 @@ endif
 .PHONY: rclone test_all vars version
 
 rclone:
-	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
+	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
 	mkdir -p `go env GOPATH`/bin/
 	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
 	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
 
 test_all:
-	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/rclone/rclone/fstest/test_all
+	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
 
 vars:
 	@echo SHELL="'$(SHELL)'"
@@ -188,10 +188,10 @@ upload_github:
 	./bin/upload-github $(TAG)
 
 cross:	doc
-	go run bin/cross-compile.go -release current $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go -release current $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 beta:
-	go run bin/cross-compile.go $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
@@ -199,23 +199,23 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 
 ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
-ifndef BRANCH_PATH
+ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifndef BRANCH_PATH
+ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
 	@echo Beta release ready at $(BETA_URL)

RELEASE.md

@@ -65,9 +65,8 @@ Now
   * git cherry-pick any fixes
   * Do the steps as above
   * make startstable
-  * NB this overwrites the current beta so we need to do this - FIXME is this true any more?
   * git co master
-  * # cherry pick the changes to the changelog
+  * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md
   * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
   * git push

VERSION

@@ -1 +1 @@
-v1.53.2
+v1.53.4

backend/chunker/chunker.go

@@ -121,6 +121,8 @@ const maxTransactionProbes = 100
 // standard chunker errors
 var (
 	ErrChunkOverflow = errors.New("chunk number overflow")
+	ErrMetaTooBig    = errors.New("metadata is too big")
+	ErrMetaUnknown   = errors.New("unknown metadata, please upgrade rclone")
 )
 
 // variants of baseMove's parameter delMode
@@ -693,43 +695,47 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 		switch entry := dirOrObject.(type) {
 		case fs.Object:
 			remote := entry.Remote()
-			if mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(remote); mainRemote != "" {
-				if xactID != "" {
-					if revealHidden {
-						fs.Infof(f, "ignore temporary chunk %q", remote)
-					}
-					break
+			mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(remote)
+			if mainRemote == "" {
+				// this is meta object or standalone file
+				object := f.newObject("", entry, nil)
+				byRemote[remote] = object
+				tempEntries = append(tempEntries, object)
+				break
+			}
+			// this is some kind of chunk
+			// metobject should have been created above if present
+			isSpecial := xactID != "" || ctrlType != ""
+			mainObject := byRemote[mainRemote]
+			if mainObject == nil && f.useMeta && !isSpecial {
+				fs.Debugf(f, "skip orphan data chunk %q", remote)
+				break
+			}
+			if mainObject == nil && !f.useMeta {
+				// this is the "nometa" case
+				// create dummy chunked object without metadata
+				mainObject = f.newObject(mainRemote, nil, nil)
+				byRemote[mainRemote] = mainObject
+				if !badEntry[mainRemote] {
+					tempEntries = append(tempEntries, mainObject)
 				}
-				if ctrlType != "" {
-					if revealHidden {
-						fs.Infof(f, "ignore control chunk %q", remote)
-					}
-					break
+			}
+			if isSpecial {
+				if revealHidden {
+					fs.Infof(f, "ignore non-data chunk %q", remote)
 				}
-				mainObject := byRemote[mainRemote]
-				if mainObject == nil && f.useMeta {
-					fs.Debugf(f, "skip chunk %q without meta object", remote)
-					break
-				}
-				if mainObject == nil {
-					// useMeta is false - create chunked object without metadata
-					mainObject = f.newObject(mainRemote, nil, nil)
-					byRemote[mainRemote] = mainObject
-					if !badEntry[mainRemote] {
-						tempEntries = append(tempEntries, mainObject)
-					}
-				}
-				if err := mainObject.addChunk(entry, chunkNo); err != nil {
-					if f.opt.FailHard {
-						return nil, err
-					}
-					badEntry[mainRemote] = true
+				// need to read metadata to ensure actual object type
+				if f.useMeta && mainObject != nil && mainObject.size <= maxMetadataSize {
+					mainObject.unsure = true
 				}
 				break
 			}
-			object := f.newObject("", entry, nil)
-			byRemote[remote] = object
-			tempEntries = append(tempEntries, object)
+			if err := mainObject.addChunk(entry, chunkNo); err != nil {
+				if f.opt.FailHard {
+					return nil, err
+				}
+				badEntry[mainRemote] = true
+			}
 		case fs.Directory:
 			isSubdir[entry.Remote()] = true
 			wrapDir := fs.NewDirCopy(ctx, entry)
@@ -784,6 +790,13 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 // but opening even a small file can be slow on some backends.
 //
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	return f.scanObject(ctx, remote, false)
+}
+
+// scanObject is like NewObject with optional quick scan mode.
+// The quick mode avoids directory requests other than `List`,
+// ignores non-chunked objects and skips chunk size checks.
+func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.Object, error) {
 	if err := f.forbidChunk(false, remote); err != nil {
 		return nil, errors.Wrap(err, "can't access")
 	}
@@ -844,8 +857,15 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 			continue // bypass regexp to save cpu
 		}
 		mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(entryRemote)
-		if mainRemote == "" || mainRemote != remote || ctrlType != "" || xactID != "" {
-			continue // skip non-conforming, temporary and control chunks
+		if mainRemote == "" || mainRemote != remote {
+			continue // skip non-conforming chunks
+		}
+		if ctrlType != "" || xactID != "" {
+			if f.useMeta {
+				// temporary/control chunk calls for lazy metadata read
+				o.unsure = true
+			}
+			continue
 		}
 		//fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
 		if err := o.addChunk(entry, chunkNo); err != nil {
@@ -855,7 +875,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 	if o.main == nil && (o.chunks == nil || len(o.chunks) == 0) {
 		// Scanning hasn't found data chunks with conforming names.
-		if f.useMeta {
+		if f.useMeta || quickScan {
 			// Metadata is required but absent and there are no chunks.
 			return nil, fs.ErrorObjectNotFound
 		}
@@ -878,8 +898,10 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	// file without metadata. Validate it and update the total data size.
 	// As an optimization, skip metadata reading here - we will call
 	// readMetadata lazily when needed (reading can be expensive).
-	if err := o.validate(); err != nil {
-		return nil, err
+	if !quickScan {
+		if err := o.validate(); err != nil {
+			return nil, err
+		}
 	}
 	return o, nil
 }
@@ -888,13 +910,24 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if o.isFull {
 		return nil
 	}
-	if !o.isComposite() || !o.f.useMeta {
+	if !o.f.useMeta || (!o.isComposite() && !o.unsure) {
 		o.isFull = true
 		return nil
 	}
 
 	// validate metadata
 	metaObject := o.main
+	if metaObject.Size() > maxMetadataSize {
+		if o.unsure {
+			// this is not metadata but a foreign object
+			o.unsure = false
+			o.chunks = nil  // make isComposite return false
+			o.isFull = true // cache results
+			return nil
+		}
+		return ErrMetaTooBig
+	}
+
 	reader, err := metaObject.Open(ctx)
 	if err != nil {
 		return err
@@ -907,8 +940,22 @@ func (o *Object) readMetadata(ctx context.Context) error {
 
 	switch o.f.opt.MetaFormat {
 	case "simplejson":
-		metaInfo, err := unmarshalSimpleJSON(ctx, metaObject, metadata, true)
-		if err != nil {
+		metaInfo, madeByChunker, err := unmarshalSimpleJSON(ctx, metaObject, metadata)
+		if o.unsure {
+			o.unsure = false
+			if !madeByChunker {
+				// this is not metadata but a foreign object
+				o.chunks = nil  // make isComposite return false
+				o.isFull = true // cache results
+				return nil
+			}
+		}
+		switch err {
+		case nil:
+			// fall thru
+		case ErrMetaTooBig, ErrMetaUnknown:
+			return err // return these errors unwrapped for unit tests
+		default:
 			return errors.Wrap(err, "invalid metadata")
 		}
 		if o.size != metaInfo.Size() || len(o.chunks) != metaInfo.nChunks {
@@ -923,7 +970,27 @@ func (o *Object) readMetadata(ctx context.Context) error {
 }
 
 // put implements Put, PutStream, PutUnchecked, Update
-func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption, basePut putFn) (obj fs.Object, err error) {
+func (f *Fs) put(
+	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
+	basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
+
+	if err := f.forbidChunk(src, remote); err != nil {
+		return nil, errors.Wrap(err, action+" refused")
+	}
+	if target == nil {
+		// Get target object with a quick directory scan
+		if obj, err := f.scanObject(ctx, remote, true); err == nil {
+			target = obj
+		}
+	}
+	if target != nil {
+		obj := target.(*Object)
+		if err := obj.readMetadata(ctx); err == ErrMetaUnknown {
+			// refuse to update a file of unsupported format
+			return nil, errors.Wrap(err, "refusing to "+action)
+		}
+	}
+
 	c := f.newChunkingReader(src)
 	wrapIn := c.wrapStream(ctx, in, src)
 
@@ -1013,8 +1080,8 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, remote st
 	// Check for input that looks like valid metadata
 	needMeta := len(c.chunks) > 1
 	if c.readCount <= maxMetadataSize && len(c.chunks) == 1 {
-		_, err := unmarshalSimpleJSON(ctx, c.chunks[0], c.smallHead, false)
-		needMeta = err == nil
+		_, madeByChunker, _ := unmarshalSimpleJSON(ctx, c.chunks[0], c.smallHead)
+		needMeta = madeByChunker
 	}
 
 	// Finalize small object as non-chunked.
@@ -1261,29 +1328,16 @@ func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
 // will return the object and the error, otherwise will return
 // nil and the error
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if err := f.forbidChunk(src, src.Remote()); err != nil {
-		return nil, errors.Wrap(err, "refusing to put")
-	}
-	return f.put(ctx, in, src, src.Remote(), options, f.base.Put)
+	return f.put(ctx, in, src, src.Remote(), options, f.base.Put, "put", nil)
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if err := f.forbidChunk(src, src.Remote()); err != nil {
-		return nil, errors.Wrap(err, "refusing to upload")
-	}
-	return f.put(ctx, in, src, src.Remote(), options, f.base.Features().PutStream)
+	return f.put(ctx, in, src, src.Remote(), options, f.base.Features().PutStream, "upload", nil)
 }
 
 // Update in to the object with the modTime given of the given size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if err := o.f.forbidChunk(o, o.Remote()); err != nil {
-		return errors.Wrap(err, "update refused")
-	}
-	if err := o.readMetadata(ctx); err != nil {
-		// refuse to update a file of unsupported format
-		return errors.Wrap(err, "refusing to update")
-	}
 	basePut := o.f.base.Put
 	if src.Size() < 0 {
 		basePut = o.f.base.Features().PutStream
@@ -1291,7 +1345,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			return errors.New("wrapped file system does not support streaming uploads")
 		}
 	}
-	oNew, err := o.f.put(ctx, in, src, o.Remote(), options, basePut)
+	oNew, err := o.f.put(ctx, in, src, o.Remote(), options, basePut, "update", o)
 	if err == nil {
 		*o = *oNew.(*Object)
 	}
@@ -1405,7 +1459,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		// to corrupt file in hard mode. Hence, refuse to Remove, too.
 		return errors.Wrap(err, "refuse to corrupt")
 	}
-	if err := o.readMetadata(ctx); err != nil {
+	if err := o.readMetadata(ctx); err == ErrMetaUnknown {
 		// Proceed but warn user that unexpected things can happen.
 		fs.Errorf(o, "Removing a file with unsupported metadata: %v", err)
 	}
@@ -1433,6 +1487,11 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 	if err := f.forbidChunk(o, remote); err != nil {
 		return nil, errors.Wrapf(err, "can't %s", opName)
 	}
+	if err := o.readMetadata(ctx); err != nil {
+		// Refuse to copy/move composite files with invalid or future
+		// metadata format which might involve unsupported chunk types.
+		return nil, errors.Wrapf(err, "can't %s this file", opName)
+	}
 	if !o.isComposite() {
 		fs.Debugf(o, "%s non-chunked object...", opName)
 		oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
@@ -1441,11 +1500,6 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		}
 		return f.newObject("", oResult, nil), nil
 	}
-	if err := o.readMetadata(ctx); err != nil {
-		// Refuse to copy/move composite files with invalid or future
-		// metadata format which might involve unsupported chunk types.
-		return nil, errors.Wrapf(err, "can't %s this file", opName)
-	}
 
 	fs.Debugf(o, "%s %d data chunks...", opName, len(o.chunks))
 	mainRemote := o.remote
@@ -1536,6 +1590,10 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string)
 		return
 	}
 
+	if obj.unsure {
+		// ensure object is composite if need to re-read metadata
+		_ = obj.readMetadata(ctx)
+	}
 	requireMetaHash := obj.isComposite() && f.opt.MetaFormat == "simplejson"
 	if !requireMetaHash && !f.hashAll {
 		ok = true // hash is not required for metadata
@@ -1719,6 +1777,7 @@ type Object struct {
 	chunks []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
 	size   int64       // cached total size of chunks in a composite file or -1 for non-chunked files
 	isFull bool        // true if metadata has been read
+	unsure bool        // true if need to read metadata to detect object type
 	md5    string
 	sha1   string
 	f      *Fs
@@ -1869,15 +1928,16 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
 // on the level of wrapped remote but chunker is unaware of that.
 //
 func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) {
+	if err := o.readMetadata(ctx); err != nil {
+		return "", err // valid metadata is required to get hash, abort
+	}
 	if !o.isComposite() {
 		// First, chain to the wrapped non-chunked file if possible.
 		if value, err := o.mainChunk().Hash(ctx, hashType); err == nil && value != "" {
 			return value, nil
 		}
 	}
-	if err := o.readMetadata(ctx); err != nil {
-		return "", err // valid metadata is required to get hash, abort
-	}
+
 	// Try hash from metadata if the file is composite or if wrapped remote fails.
 	switch hashType {
 	case hash.MD5:
@@ -1902,13 +1962,13 @@ func (o *Object) UnWrap() fs.Object {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
-	if !o.isComposite() {
-		return o.mainChunk().Open(ctx, options...) // chain to wrapped non-chunked file
-	}
 	if err := o.readMetadata(ctx); err != nil {
 		// refuse to open unsupported format
 		return nil, errors.Wrap(err, "can't open")
 	}
+	if !o.isComposite() {
+		return o.mainChunk().Open(ctx, options...) // chain to wrapped non-chunked file
+	}
 
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
@@ -2181,57 +2241,57 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1 s
 // handled by current implementation.
 // The version check below will then explicitly ask user to upgrade rclone.
 //
-func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte, strictChecks bool) (info *ObjectInfo, err error) {
+func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) {
 	// Be strict about JSON format
 	// to reduce possibility that a random small file resembles metadata.
 	if data != nil && len(data) > maxMetadataSize {
-		return nil, errors.New("too big")
+		return nil, false, ErrMetaTooBig
 	}
 	if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
-		return nil, errors.New("invalid json")
+		return nil, false, errors.New("invalid json")
 	}
 	var metadata metaSimpleJSON
 	err = json.Unmarshal(data, &metadata)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	// Basic fields are strictly required
 	// to reduce possibility that a random small file resembles metadata.
 	if metadata.Version == nil || metadata.Size == nil || metadata.ChunkNum == nil {
-		return nil, errors.New("missing required field")
+		return nil, false, errors.New("missing required field")
 	}
 	// Perform strict checks, avoid corruption of future metadata formats.
 	if *metadata.Version < 1 {
-		return nil, errors.New("wrong version")
+		return nil, false, errors.New("wrong version")
 	}
 	if *metadata.Size < 0 {
-		return nil, errors.New("negative file size")
+		return nil, false, errors.New("negative file size")
 	}
 	if *metadata.ChunkNum < 0 {
-		return nil, errors.New("negative number of chunks")
+		return nil, false, errors.New("negative number of chunks")
 	}
 	if *metadata.ChunkNum > maxSafeChunkNumber {
-		return nil, ErrChunkOverflow
+		return nil, true, ErrChunkOverflow // produced by incompatible version of rclone
 	}
 	if metadata.MD5 != "" {
 		_, err = hex.DecodeString(metadata.MD5)
 		if len(metadata.MD5) != 32 || err != nil {
-			return nil, errors.New("wrong md5 hash")
+			return nil, false, errors.New("wrong md5 hash")
 		}
 	}
 	if metadata.SHA1 != "" {
 		_, err = hex.DecodeString(metadata.SHA1)
 		if len(metadata.SHA1) != 40 || err != nil {
-			return nil, errors.New("wrong sha1 hash")
+			return nil, false, errors.New("wrong sha1 hash")
 		}
 	}
 	// ChunkNum is allowed to be 0 in future versions
 	if *metadata.ChunkNum < 1 && *metadata.Version <= metadataVersion {
-		return nil, errors.New("wrong number of chunks")
+		return nil, false, errors.New("wrong number of chunks")
 	}
 	// Non-strict mode also accepts future metadata versions
-	if *metadata.Version > metadataVersion && strictChecks {
-		return nil, fmt.Errorf("version %d is not supported, please upgrade rclone", metadata.Version)
+	if *metadata.Version > metadataVersion {
+		return nil, true, ErrMetaUnknown // produced by incompatible version of rclone
 	}
 
 	var nilFs *Fs // nil object triggers appropriate type method
@@ -2239,7 +2299,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte,
 	info.nChunks = *metadata.ChunkNum
 	info.md5 = metadata.MD5
 	info.sha1 = metadata.SHA1
-	return info, nil
+	return info, true, nil
 }
 
 func silentlyRemove(ctx context.Context, o fs.Object) {

backend/googlecloudstorage/googlecloudstorage.go

@@ -841,20 +841,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		remote: remote,
 	}
 
-	var newObject *storage.Object
-	err = f.pacer.Call(func() (bool, error) {
-		copyObject := f.svc.Objects.Copy(srcBucket, srcPath, dstBucket, dstPath, nil)
-		if !f.opt.BucketPolicyOnly {
-			copyObject.DestinationPredefinedAcl(f.opt.ObjectACL)
+	rewriteRequest := f.svc.Objects.Rewrite(srcBucket, srcPath, dstBucket, dstPath, nil)
+	if !f.opt.BucketPolicyOnly {
+		rewriteRequest.DestinationPredefinedAcl(f.opt.ObjectACL)
+	}
+	var rewriteResponse *storage.RewriteResponse
+	for {
+		err = f.pacer.Call(func() (bool, error) {
+			rewriteResponse, err = rewriteRequest.Context(ctx).Do()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return nil, err
 		}
-		newObject, err = copyObject.Context(ctx).Do()
-		return shouldRetry(err)
-	})
-	if err != nil {
-		return nil, err
+		if rewriteResponse.Done {
+			break
+		}
+		rewriteRequest.RewriteToken(rewriteResponse.RewriteToken)
+		fs.Debugf(dstObj, "Continuing rewrite %d bytes done", rewriteResponse.TotalBytesRewritten)
 	}
 	// Set the metadata for the new object while we have it
-	dstObj.setMetaData(newObject)
+	dstObj.setMetaData(rewriteResponse.Resource)
 	return dstObj, nil
 }
 

backend/jottacloud/jottacloud.go

@@ -730,6 +730,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
 		_, err := f.readMetaDataForPath(ctx, "")
+		if err == fs.ErrorNotAFile {
+			err = nil
+		}
 		return err
 	})
 
@@ -1466,6 +1469,8 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	o.fs.tokenRenewer.Start()
+	defer o.fs.tokenRenewer.Stop()
 	size := src.Size()
 	md5String, err := src.Hash(ctx, hash.MD5)
 	if err != nil || md5String == "" {

backend/pcloud/api/types.go

@@ -104,8 +104,9 @@ type ItemResult struct {
 
 // Hashes contains the supported hashes
 type Hashes struct {
-	SHA1 string `json:"sha1"`
-	MD5  string `json:"md5"`
+	SHA1   string `json:"sha1"`
+	MD5    string `json:"md5"`
+	SHA256 string `json:"sha256"`
 }
 
 // UploadFileResponse is the response from /uploadfile

backend/pcloud/pcloud.go

@@ -885,6 +885,13 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
+	// EU region supports SHA1 and SHA256 (but rclone doesn't
+	// support SHA256 yet).
+	//
+	// https://forum.rclone.org/t/pcloud-to-local-no-hashes-in-common/19440
+	if f.opt.Hostname == "eapi.pcloud.com" {
+		return hash.Set(hash.SHA1)
+	}
 	return hash.Set(hash.MD5 | hash.SHA1)
 }
 

backend/union/policy/eprand.go

@@ -3,7 +3,6 @@ package policy
 import (
 	"context"
 	"math/rand"
-	"time"
 
 	"github.com/rclone/rclone/backend/union/upstream"
 	"github.com/rclone/rclone/fs"
@@ -20,12 +19,10 @@ type EpRand struct {
 }
 
 func (p *EpRand) rand(upstreams []*upstream.Fs) *upstream.Fs {
-	rand.Seed(time.Now().Unix())
 	return upstreams[rand.Intn(len(upstreams))]
 }
 
 func (p *EpRand) randEntries(entries []upstream.Entry) upstream.Entry {
-	rand.Seed(time.Now().Unix())
 	return entries[rand.Intn(len(entries))]
 }
 

backend/webdav/webdav.go

@@ -1129,10 +1129,14 @@ func (o *Object) Storable() bool {
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var resp *http.Response
+	fs.FixRangeOption(options, o.size)
 	opts := rest.Opts{
 		Method:  "GET",
 		Path:    o.filePath(),
 		Options: options,
+		ExtraHeaders: map[string]string{
+			"Depth": "0",
+		},
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(ctx, &opts)

bin/cross-compile.go

@@ -36,6 +36,7 @@ var (
 	cgo         = flag.Bool("cgo", false, "Use cgo for the build")
 	noClean     = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
 	tags        = flag.String("tags", "", "Space separated list of build tags")
+	buildmode   = flag.String("buildmode", "", "Passed to go build -buildmode flag")
 	compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
 )
 
@@ -300,8 +301,15 @@ func compileArch(version, goos, goarch, dir string) bool {
 		"-trimpath",
 		"-o", output,
 		"-tags", *tags,
-		"..",
 	}
+	if *buildmode != "" {
+		args = append(args,
+			"-buildmode", *buildmode,
+		)
+	}
+	args = append(args,
+		"..",
+	)
 	env := []string{
 		"GOOS=" + goos,
 		"GOARCH=" + stripVersion(goarch),

bin/nfpm.yaml

@@ -15,10 +15,12 @@ description: |
 vendor: "rclone"
 homepage: "https://rclone.org"
 license: "MIT"
-# No longer supported? See https://github.com/goreleaser/nfpm/issues/144
-# bindir: "/usr/bin"
-files:
-  ./rclone: "/usr/bin/rclone"
-  ./README.html: "/usr/share/doc/rclone/README.html"
-  ./README.txt: "/usr/share/doc/rclone/README.txt"
-  ./rclone.1: "/usr/share/man/man1/rclone.1"
+contents:
+  - src: ./rclone
+    dst: /usr/bin/rclone
+  - src: ./README.html
+    dst: /usr/share/doc/rclone/README.html
+  - src: ./README.txt
+    dst: /usr/share/doc/rclone/README.txt
+  - src: ./rclone.1
+    dst: /usr/share/man/man1/rclone.1

cmd/cmd.go

@@ -9,7 +9,6 @@ package cmd
 import (
 	"fmt"
 	"log"
-	"math/rand"
 	"os"
 	"os/exec"
 	"path"
@@ -35,6 +34,7 @@ import (
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -512,7 +512,9 @@ func AddBackendFlags() {
 
 // Main runs rclone interpreting flags and commands out of os.Args
 func Main() {
-	rand.Seed(time.Now().Unix())
+	if err := random.Seed(); err != nil {
+		log.Fatalf("Fatal error: %v", err)
+	}
 	setupRootCommand(Root)
 	AddBackendFlags()
 	if err := Root.Execute(); err != nil {

cmd/cmount/mount_brew.go

@@ -0,0 +1,32 @@
+// Build for macos with the brew tag to handle the absence
+// of fuse and print an appropriate error message
+
+// +build brew
+// +build darwin
+
+package cmount
+
+import (
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/cmd/mountlib"
+	"github.com/rclone/rclone/vfs"
+)
+
+func init() {
+	name := "mount"
+	cmd := mountlib.NewMountCommand(name, false, mount)
+	cmd.Aliases = append(cmd.Aliases, "cmount")
+	mountlib.AddRc("cmount", mount)
+}
+
+// mount the file system
+//
+// The mount point will be ready when this returns.
+//
+// returns an error, and an error channel for the serve process to
+// report an error when fusermount is called.
+func mount(_ *vfs.VFS, _ string, _ *mountlib.Options) (<-chan error, func() error, error) {
+	return nil, nil, errors.New("mount is not supported on MacOS when installed via Homebrew. " +
+		"Please install the binaries available at https://rclone." +
+		"org/downloads/ instead if you want to use the mount command")
+}

cmd/cmount/mount_unsupported.go

@@ -1,6 +1,6 @@
 // Build for cmount for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build !linux,!darwin,!freebsd,!windows !cgo !cmount
+// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
 
 package cmount

cmd/mount/dir.go

@@ -107,6 +107,13 @@ func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error)
 	if err != nil {
 		return nil, translateError(err)
 	}
+	dirents = append(dirents, fuse.Dirent{
+		Type: fuse.DT_Dir,
+		Name: ".",
+	}, fuse.Dirent{
+		Type: fuse.DT_Dir,
+		Name: "..",
+	})
 	for _, node := range items {
 		name := node.Name()
 		if len(name) > mountlib.MaxLeafSize {

cmd/mount/mount_unsupported.go

@@ -1,7 +1,7 @@
 // Build for mount for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// Invert the build constraint: linux,go1.13 darwin,go1.13 freebsd,go1.13
+// Invert the build constraint: linux,go1.13 freebsd,go1.13
 //
 // !((linux&&go1.13) || (darwin&&go1.13) || (freebsd&&go1.13))
 // == !(linux&&go1.13) && !(darwin&&go1.13) && !(freebsd&&go1.13))

cmd/serve/http/http.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"io"
 	"net/http"
 	"os"
 	"path"
@@ -172,8 +173,11 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
 	obj := entry.(fs.Object)
 	file := node.(*vfs.File)
 
-	// Set content length since we know how long the object is
-	w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
+	// Set content length if we know how long the object is
+	knownSize := obj.Size() >= 0
+	if knownSize {
+		w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
+	}
 
 	// Set content type
 	mimeType := fs.MimeType(r.Context(), obj)
@@ -210,5 +214,19 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
 	// FIXME in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
 
 	// Serve the file
-	http.ServeContent(w, r, remote, node.ModTime(), in)
+	if knownSize {
+		http.ServeContent(w, r, remote, node.ModTime(), in)
+	} else {
+		// http.ServeContent can't serve unknown length files
+		if rangeRequest := r.Header.Get("Range"); rangeRequest != "" {
+			http.Error(w, "Can't use Range: on files of unknown length", http.StatusRequestedRangeNotSatisfiable)
+			return
+		}
+		n, err := io.Copy(w, in)
+		if err != nil {
+			fs.Errorf(obj, "Didn't finish writing GET request (wrote %d/unknown bytes): %v", n, err)
+			return
+		}
+	}
+
 }

cmd/serve/sftp/server.go

@@ -75,6 +75,39 @@ func (s *server) getVFS(what string, sshConn *ssh.ServerConn) (VFS *vfs.VFS) {
 	return VFS
 }
 
+// Accept a single connection - run in a go routine as the ssh
+// authentication can block
+func (s *server) acceptConnection(nConn net.Conn) {
+	what := describeConn(nConn)
+
+	// Before use, a handshake must be performed on the incoming net.Conn.
+	sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
+	if err != nil {
+		fs.Errorf(what, "SSH login failed: %v", err)
+		return
+	}
+
+	fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
+
+	// Discard all global out-of-band Requests
+	go ssh.DiscardRequests(reqs)
+
+	c := &conn{
+		what: what,
+		vfs:  s.getVFS(what, sshConn),
+	}
+	if c.vfs == nil {
+		fs.Infof(what, "Closing unauthenticated connection (couldn't find VFS)")
+		_ = nConn.Close()
+		return
+	}
+	c.handlers = newVFSHandler(c.vfs)
+
+	// Accept all channels
+	go c.handleChannels(chans)
+}
+
+// Accept connections and call them in a go routine
 func (s *server) acceptConnections() {
 	for {
 		nConn, err := s.listener.Accept()
@@ -85,33 +118,7 @@ func (s *server) acceptConnections() {
 			fs.Errorf(nil, "Failed to accept incoming connection: %v", err)
 			continue
 		}
-		what := describeConn(nConn)
-
-		// Before use, a handshake must be performed on the incoming net.Conn.
-		sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
-		if err != nil {
-			fs.Errorf(what, "SSH login failed: %v", err)
-			continue
-		}
-
-		fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
-
-		// Discard all global out-of-band Requests
-		go ssh.DiscardRequests(reqs)
-
-		c := &conn{
-			what: what,
-			vfs:  s.getVFS(what, sshConn),
-		}
-		if c.vfs == nil {
-			fs.Infof(what, "Closing unauthenticated connection (couldn't find VFS)")
-			_ = nConn.Close()
-			continue
-		}
-		c.handlers = newVFSHandler(c.vfs)
-
-		// Accept all channels
-		go c.handleChannels(chans)
+		go s.acceptConnection(nConn)
 	}
 }
 

docs/content/changelog.md

@@ -5,6 +5,20 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.53.3 - 2020-11-19
+
+[See commits](https://github.com/rclone/rclone/compare/v1.53.2...v1.53.3)
+
+* Bug Fixes
+    * random: Fix incorrect use of math/rand instead of crypto/rand CVE-2020-28924 (Nick Craig-Wood)
+        * Passwords you have generated with `rclone config` may be insecure
+        * See [issue #4783](https://github.com/rclone/rclone/issues/4783) for more details and a checking tool
+    * random: Seed math/rand in one place with crypto strong seed (Nick Craig-Wood)
+* VFS
+    * Fix vfs/refresh calls with fs= parameter (Nick Craig-Wood)
+* Sharefile
+    * Fix backend due to API swapping integers for strings (Nick Craig-Wood)
+
 ## v1.53.2 - 2020-10-26
 
 [See commits](https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)

docs/content/flags.md

@@ -147,7 +147,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format.
       --use-mmap                             Use mmap allocator (see docs).
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.2")
+      --user-agent string                    Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.3")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 

docs/content/overview.md

@@ -22,7 +22,7 @@ Here is an overview of the major features of each cloud storage system.
 | Backblaze B2                 | SHA1        | Yes     | No               | No              | R/W       |
 | Box                          | SHA1        | Yes     | Yes              | No              | -         |
 | Citrix ShareFile             | MD5         | Yes     | Yes              | No              | -         |
-| Dropbox                      | DBHASH †    | Yes     | Yes              | No              | -         |
+| Dropbox                      | DBHASH ¹    | Yes     | Yes              | No              | -         |
 | FTP                          | -           | No      | No               | No              | -         |
 | Google Cloud Storage         | MD5         | Yes     | No               | No              | R/W       |
 | Google Drive                 | MD5         | Yes     | No               | Yes             | R/W       |
@@ -31,25 +31,52 @@ Here is an overview of the major features of each cloud storage system.
 | Hubic                        | MD5         | Yes     | No               | No              | R/W       |
 | Jottacloud                   | MD5         | Yes     | Yes              | No              | R/W       |
 | Koofr                        | MD5         | No      | Yes              | No              | -         |
-| Mail.ru Cloud                | Mailru ‡‡‡  | Yes     | Yes              | No              | -         |
+| Mail.ru Cloud                | Mailru ⁶    | Yes     | Yes              | No              | -         |
 | Mega                         | -           | No      | No               | Yes             | -         |
 | Memory                       | MD5         | Yes     | No               | No              | -         |
 | Microsoft Azure Blob Storage | MD5         | Yes     | No               | No              | R/W       |
-| Microsoft OneDrive           | SHA1 ‡‡     | Yes     | Yes              | No              | R         |
-| OpenDrive                    | MD5         | Yes     | Yes              | Partial \*      | -         |
+| Microsoft OneDrive           | SHA1 ⁵      | Yes     | Yes              | No              | R         |
+| OpenDrive                    | MD5         | Yes     | Yes              | Partial ⁸       | -         |
 | OpenStack Swift              | MD5         | Yes     | No               | No              | R/W       |
-| pCloud                       | MD5, SHA1   | Yes     | No               | No              | W         |
+| pCloud                       | MD5, SHA1 ⁷ | Yes     | No               | No              | W         |
 | premiumize.me                | -           | No      | Yes              | No              | R         |
 | put.io                       | CRC-32      | Yes     | No               | Yes             | R         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
 | Seafile                      | -           | No      | No               | No              | -         |
-| SFTP                         | MD5, SHA1 ‡ | Yes     | Depends          | No              | -         |
+| SFTP                         | MD5, SHA1 ² | Yes     | Depends          | No              | -         |
 | SugarSync                    | -           | No      | No               | No              | -         |
 | Tardigrade                   | -           | Yes     | No               | No              | -         |
-| WebDAV                       | MD5, SHA1 ††| Yes ††† | Depends          | No              | -         |
+| WebDAV                       | MD5, SHA1 ³ | Yes ⁴   | Depends          | No              | -         |
 | Yandex Disk                  | MD5         | Yes     | No               | No              | R/W       |
 | The local filesystem         | All         | Yes     | Depends          | No              | -         |
 
+### Notes
+
+¹ Dropbox supports [its own custom
+hash](https://www.dropbox.com/developers/reference/content-hash).
+This is an SHA256 sum of all the 4MB block SHA256s.
+
+² SFTP supports checksums if the same login has shell access and
+`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
+
+³ WebDAV supports hashes when used with Owncloud and Nextcloud only.
+
+⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only.
+
+⁵ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
+for business and SharePoint server support Microsoft's own
+[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
+
+⁶ Mail.ru uses its own modified SHA1 hash
+
+⁷ pCloud only supports SHA1 (not MD5) in its EU region
+
+⁸ Opendrive does not support creation of duplicate files using
+their web client interface or other stock clients, but the underlying
+storage platform has been determined to allow duplicate files, and it
+is possible to create them with `rclone`.  It may be that this is a
+mistake or an unsupported feature.
+
 ### Hash ###
 
 The cloud storage system supports various hash types of the objects.
@@ -60,23 +87,6 @@ the `check` command.
 To use the verify checksums when transferring between cloud storage
 systems they must support a common hash type.
 
-† Note that Dropbox supports [its own custom
-hash](https://www.dropbox.com/developers/reference/content-hash).
-This is an SHA256 sum of all the 4MB block SHA256s.
-
-‡ SFTP supports checksums if the same login has shell access and `md5sum`
-or `sha1sum` as well as `echo` are in the remote's PATH.
-
-†† WebDAV supports hashes when used with Owncloud and Nextcloud only.
-
-††† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
-
-‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
-for business and SharePoint server support Microsoft's own
-[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
-
-‡‡‡ Mail.ru uses its own modified SHA1 hash
-
 ### ModTime ###
 
 The cloud storage system supports setting modification times on
@@ -117,12 +127,6 @@ objects with the same name.
 This confuses rclone greatly when syncing - use the `rclone dedupe`
 command to rename or remove duplicates.
 
-\* Opendrive does not support creation of duplicate files using
-their web client interface or other stock clients, but the underlying
-storage platform has been determined to allow duplicate files, and it
-is possible to create them with `rclone`.  It may be that this is a
-mistake or an unsupported feature.
-
 ### Restricted filenames ###
 
 Some cloud storage systems might have restrictions on the characters

docs/content/pcloud.md

@@ -90,8 +90,11 @@ second.  These will be used to detect whether objects need syncing or
 not.  In order to set a Modification time pCloud requires the object
 be re-uploaded.
 
-pCloud supports MD5 and SHA1 type hashes, so you can use the
-`--checksum` flag.
+pCloud supports MD5 and SHA1 type hashes in the US region but and SHA1
+only in the EU region, so you can use the `--checksum` flag.
+
+(Note that pCloud also support SHA256 in the EU region, but rclone
+does not have support for that yet.)
 
 #### Restricted filename characters
 

docs/layouts/partials/version.html

@@ -1 +1 @@
-v1.53.2
+v1.53.4

fs/accounting/stats.go

@@ -337,6 +337,8 @@ func (s *StatsInfo) String() string {
 // Transferred returns list of all completed transfers including checked and
 // failed ones.
 func (s *StatsInfo) Transferred() []TransferSnapshot {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
 	ts := make([]TransferSnapshot, 0, len(s.startedTransfers))
 
 	for _, tr := range s.startedTransfers {

fs/fshttp/http.go

@@ -31,6 +31,7 @@ var (
 	noTransport  = new(sync.Once)
 	tpsBucket    *rate.Limiter // for limiting number of http transactions per second
 	cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	logMutex     sync.Mutex
 )
 
 // StartHTTPTokenBucket starts the token bucket if necessary
@@ -328,15 +329,18 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		if t.dump&fs.DumpAuth == 0 {
 			buf = cleanAuths(buf)
 		}
+		logMutex.Lock()
 		fs.Debugf(nil, "%s", separatorReq)
 		fs.Debugf(nil, "%s (req %p)", "HTTP REQUEST", req)
 		fs.Debugf(nil, "%s", string(buf))
 		fs.Debugf(nil, "%s", separatorReq)
+		logMutex.Unlock()
 	}
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response
 	if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
+		logMutex.Lock()
 		fs.Debugf(nil, "%s", separatorResp)
 		fs.Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
 		if err != nil {
@@ -346,6 +350,7 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 			fs.Debugf(nil, "%s", string(buf))
 		}
 		fs.Debugf(nil, "%s", separatorResp)
+		logMutex.Unlock()
 	}
 	if err == nil {
 		checkServerTime(req, resp)

fs/parseduration_test.go

@@ -37,9 +37,9 @@ func TestParseDuration(t *testing.T) {
 		{"1x", 0, true},
 		{"off", time.Duration(DurationOff), false},
 		{"1h2m3s", time.Hour + 2*time.Minute + 3*time.Second, false},
-		{"2001-02-03", time.Since(time.Date(2001, 2, 3, 0, 0, 0, 0, time.Local)), false},
-		{"2001-02-03 10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local)), false},
-		{"2001-02-03T10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local)), false},
+		{"2001-02-03", time.Since(time.Date(2001, 2, 3, 0, 0, 0, 0, time.UTC)), false},
+		{"2001-02-03 10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.UTC)), false},
+		{"2001-02-03T10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.UTC)), false},
 		{"2001-02-03T10:11:12.123Z", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 123, time.UTC)), false},
 	} {
 		duration, err := ParseDuration(test.in)

fs/rc/internal.go

@@ -353,17 +353,22 @@ func init() {
 - command - a string with the command name
 - arg - a list of arguments for the backend command
 - opt - a map of string to string of options
+- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
+    - defaults to "COMBINED_OUTPUT" if not set
+    - the STREAM returnTypes will write the output to the body of the HTTP message
+    - the COMBINED_OUTPUT will write the output to the "result" parameter
 
 Returns
 
 - result - result from the backend command
+    - only set when using returnType "COMBINED_OUTPUT"
 - error	 - set if rclone exits with an error code
-- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT". "STREAM_ONLY_STDERR")
+- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
 
 For example
 
     rclone rc core/command command=ls -a mydrive:/ -o max-depth=1
-	rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
+    rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
 
 Returns
 
@@ -386,7 +391,6 @@ OR
 
 // rcRunCommand runs an rclone command with the given args and flags
 func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
-
 	command, err := in.GetString("command")
 	if err != nil {
 		command = ""
@@ -409,7 +413,7 @@ func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
 		returnType = "COMBINED_OUTPUT"
 	}
 
-	var httpResponse *http.ResponseWriter
+	var httpResponse http.ResponseWriter
 	httpResponse, err = in.GetHTTPResponseWriter()
 	if err != nil {
 		return nil, errors.Errorf("response object is required\n" + err.Error())
@@ -460,12 +464,14 @@ func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
 			"error":  false,
 		}, nil
 	} else if returnType == "STREAM_ONLY_STDOUT" {
-		cmd.Stdout = *httpResponse
+		cmd.Stdout = httpResponse
 	} else if returnType == "STREAM_ONLY_STDERR" {
-		cmd.Stderr = *httpResponse
+		cmd.Stderr = httpResponse
 	} else if returnType == "STREAM" {
-		cmd.Stdout = *httpResponse
-		cmd.Stderr = *httpResponse
+		cmd.Stdout = httpResponse
+		cmd.Stderr = httpResponse
+	} else {
+		return nil, errors.Errorf("Unknown returnType %q", returnType)
 	}
 
 	err = cmd.Run()

fs/rc/internal_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,6 +23,12 @@ func TestMain(m *testing.M) {
 		fmt.Printf("rclone %s\n", fs.Version)
 		os.Exit(0)
 	}
+	// Pretend to error if we have an unknown command
+	if os.Args[len(os.Args)-1] == "unknown_command" {
+		fmt.Printf("rclone %s\n", fs.Version)
+		fmt.Fprintf(os.Stderr, "Unknown command\n")
+		os.Exit(1)
+	}
 	os.Exit(m.Run())
 }
 
@@ -136,17 +143,56 @@ func TestCoreQuit(t *testing.T) {
 func TestCoreCommand(t *testing.T) {
 	call := Calls.Get("core/command")
 
-	var httpResponse http.ResponseWriter = httptest.NewRecorder()
+	test := func(command string, returnType string, wantOutput string, fail bool) {
+		var rec = httptest.NewRecorder()
+		var w http.ResponseWriter = rec
 
-	in := Params{
-		"command":   "version",
-		"opt":       map[string]string{},
-		"arg":       []string{},
-		"_response": &httpResponse,
+		in := Params{
+			"command":   command,
+			"opt":       map[string]string{},
+			"arg":       []string{},
+			"_response": w,
+		}
+		if returnType != "" {
+			in["returnType"] = returnType
+		} else {
+			returnType = "COMBINED_OUTPUT"
+		}
+		stream := strings.HasPrefix(returnType, "STREAM")
+		got, err := call.Fn(context.Background(), in)
+		if stream && fail {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+
+		if !stream {
+			assert.Equal(t, wantOutput, got["result"])
+			assert.Equal(t, fail, got["error"])
+		} else {
+			assert.Equal(t, wantOutput, rec.Body.String())
+		}
+		assert.Equal(t, http.StatusOK, rec.Result().StatusCode)
 	}
-	got, err := call.Fn(context.Background(), in)
-	require.NoError(t, err)
 
-	assert.Equal(t, fmt.Sprintf("rclone %s\n", fs.Version), got["result"])
-	assert.Equal(t, false, got["error"])
+	version := fmt.Sprintf("rclone %s\n", fs.Version)
+	errorString := "Unknown command\n"
+	t.Run("OK", func(t *testing.T) {
+		test("version", "", version, false)
+	})
+	t.Run("Fail", func(t *testing.T) {
+		test("unknown_command", "", version+errorString, true)
+	})
+	t.Run("Combined", func(t *testing.T) {
+		test("unknown_command", "COMBINED_OUTPUT", version+errorString, true)
+	})
+	t.Run("Stderr", func(t *testing.T) {
+		test("unknown_command", "STREAM_ONLY_STDERR", errorString, true)
+	})
+	t.Run("Stdout", func(t *testing.T) {
+		test("unknown_command", "STREAM_ONLY_STDOUT", version, true)
+	})
+	t.Run("Stream", func(t *testing.T) {
+		test("unknown_command", "STREAM", version+errorString, true)
+	})
 }

fs/rc/params.go

@@ -79,6 +79,15 @@ func Reshape(out interface{}, in interface{}) error {
 	return nil
 }
 
+// Copy shallow copies the Params
+func (p Params) Copy() (out Params) {
+	out = make(Params, len(p))
+	for k, v := range p {
+		out[k] = v
+	}
+	return out
+}
+
 // Get gets a parameter from the input
 //
 // If the parameter isn't found then error will be of type
@@ -112,15 +121,15 @@ func (p Params) GetHTTPRequest() (*http.Request, error) {
 //
 // If the parameter isn't found then error will be of type
 // ErrParamNotFound and the returned value will be nil.
-func (p Params) GetHTTPResponseWriter() (*http.ResponseWriter, error) {
+func (p Params) GetHTTPResponseWriter() (http.ResponseWriter, error) {
 	key := "_response"
 	value, err := p.Get(key)
 	if err != nil {
 		return nil, err
 	}
-	request, ok := value.(*http.ResponseWriter)
+	request, ok := value.(http.ResponseWriter)
 	if !ok {
-		return nil, ErrParamInvalid{errors.Errorf("expecting *http.ResponseWriter value for key %q (was %T)", key, value)}
+		return nil, ErrParamInvalid{errors.Errorf("expecting http.ResponseWriter value for key %q (was %T)", key, value)}
 	}
 	return request, nil
 }

fs/rc/params_test.go

@@ -2,6 +2,8 @@ package rc
 
 import (
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -61,6 +63,19 @@ func TestReshape(t *testing.T) {
 
 }
 
+func TestParamsCopy(t *testing.T) {
+	in := Params{
+		"ok":  1,
+		"x":   "seventeen",
+		"nil": nil,
+	}
+	out := in.Copy()
+	assert.Equal(t, in, out)
+	if &in == &out {
+		t.Error("didn't copy")
+	}
+}
+
 func TestParamsGet(t *testing.T) {
 	in := Params{
 		"ok": 1,
@@ -346,3 +361,53 @@ func TestParamsGetStructMissingOK(t *testing.T) {
 	assert.Equal(t, 4.2, out.Float)
 	assert.Equal(t, true, IsErrParamInvalid(e3), e3.Error())
 }
+
+func TestParamsGetHTTPRequest(t *testing.T) {
+	in := Params{}
+	req, err := in.GetHTTPRequest()
+	assert.Nil(t, req)
+	assert.Error(t, err)
+	assert.Equal(t, true, IsErrParamNotFound(err), err.Error())
+
+	in = Params{
+		"_request": 42,
+	}
+	req, err = in.GetHTTPRequest()
+	assert.Nil(t, req)
+	assert.Error(t, err)
+	assert.Equal(t, true, IsErrParamInvalid(err), err.Error())
+
+	r := new(http.Request)
+	in = Params{
+		"_request": r,
+	}
+	req, err = in.GetHTTPRequest()
+	assert.NotNil(t, req)
+	assert.NoError(t, err)
+	assert.Equal(t, r, req)
+}
+
+func TestParamsGetHTTPResponseWriter(t *testing.T) {
+	in := Params{}
+	wr, err := in.GetHTTPResponseWriter()
+	assert.Nil(t, wr)
+	assert.Error(t, err)
+	assert.Equal(t, true, IsErrParamNotFound(err), err.Error())
+
+	in = Params{
+		"_response": 42,
+	}
+	wr, err = in.GetHTTPResponseWriter()
+	assert.Nil(t, wr)
+	assert.Error(t, err)
+	assert.Equal(t, true, IsErrParamInvalid(err), err.Error())
+
+	var w http.ResponseWriter = httptest.NewRecorder()
+	in = Params{
+		"_response": w,
+	}
+	wr, err = in.GetHTTPResponseWriter()
+	assert.NotNil(t, wr)
+	assert.NoError(t, err)
+	assert.Equal(t, w, wr)
+}

fs/rc/rcserver/rcserver.go

@@ -183,7 +183,7 @@ func writeError(path string, in rc.Params, w http.ResponseWriter, err error, sta
 	})
 	if err != nil {
 		// can't return the error at this point
-		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		fs.Errorf(nil, "rc: writeError: failed to write JSON output from %#v: %v", in, err)
 	}
 }
 
@@ -267,6 +267,9 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 		writeError(path, in, w, errors.Errorf("authentication must be set up on the rc server to use %q or the --rc-no-auth flag must be in use", path), http.StatusForbidden)
 		return
 	}
+
+	inOrig := in.Copy()
+
 	if call.NeedsRequest {
 		// Add the request to RC
 		in["_request"] = r
@@ -279,7 +282,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 	// Check to see if it is async or not
 	isAsync, err := in.GetBool("_async")
 	if rc.NotErrParamNotFound(err) {
-		writeError(path, in, w, err, http.StatusBadRequest)
+		writeError(path, inOrig, w, err, http.StatusBadRequest)
 		return
 	}
 	delete(in, "_async") // remove the async parameter after parsing so vfs operations don't get confused
@@ -294,7 +297,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 		w.Header().Add("x-rclone-jobid", fmt.Sprintf("%d", jobID))
 	}
 	if err != nil {
-		writeError(path, in, w, err, http.StatusInternalServerError)
+		writeError(path, inOrig, w, err, http.StatusInternalServerError)
 		return
 	}
 	if out == nil {
@@ -305,8 +308,8 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 	err = rc.WriteJSON(w, out)
 	if err != nil {
 		// can't return the error at this point - but have a go anyway
-		writeError(path, in, w, err, http.StatusInternalServerError)
-		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		writeError(path, inOrig, w, err, http.StatusInternalServerError)
+		fs.Errorf(nil, "rc: handlePost: failed to write JSON output: %v", err)
 	}
 }
 
@@ -387,18 +390,20 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 		s.serveRoot(w, r)
 		return
 	case s.files != nil:
-		pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path)
+		if s.opt.WebUI {
+			pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path)
 
-		if s.opt.WebUI && pluginsMatchResult != nil && len(pluginsMatchResult) > 2 {
-			ok := webgui.ServePluginOK(w, r, pluginsMatchResult)
-			if !ok {
-				r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3])
-				s.pluginsHandler.ServeHTTP(w, r)
+			if pluginsMatchResult != nil && len(pluginsMatchResult) > 2 {
+				ok := webgui.ServePluginOK(w, r, pluginsMatchResult)
+				if !ok {
+					r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3])
+					s.pluginsHandler.ServeHTTP(w, r)
+					return
+				}
+				return
+			} else if webgui.ServePluginWithReferrerOK(w, r, path) {
 				return
 			}
-			return
-		} else if s.opt.WebUI && webgui.ServePluginWithReferrerOK(w, r, path) {
-			return
 		}
 		// Serve the files
 		r.URL.Path = "/" + path

fs/rc/rcserver/rcserver_test.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -16,6 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/rc"
 )
@@ -27,6 +29,21 @@ const (
 	remoteURL       = "[" + testFs + "]/" // initial URL path to fetch from that remote
 )
 
+func TestMain(m *testing.M) {
+	// Pretend to be rclone version if we have a version string parameter
+	if os.Args[len(os.Args)-1] == "version" {
+		fmt.Printf("rclone %s\n", fs.Version)
+		os.Exit(0)
+	}
+	// Pretend to error if we have an unknown command
+	if os.Args[len(os.Args)-1] == "unknown_command" {
+		fmt.Printf("rclone %s\n", fs.Version)
+		fmt.Fprintf(os.Stderr, "Unknown command\n")
+		os.Exit(1)
+	}
+	os.Exit(m.Run())
+}
+
 // Test the RC server runs and we can do HTTP fetches from it.
 // We'll do the majority of the testing with the httptest framework
 func TestRcServer(t *testing.T) {
@@ -455,6 +472,73 @@ func TestRC(t *testing.T) {
 	testServer(t, tests, &opt)
 }
 
+func TestRCWithAuth(t *testing.T) {
+	tests := []testRun{{
+		Name:        "core-command",
+		URL:         "core/command",
+		Method:      "POST",
+		Body:        `command=version`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusOK,
+		Expected: fmt.Sprintf(`{
+	"error": false,
+	"result": "rclone %s\n"
+}
+`, fs.Version),
+	}, {
+		Name:        "core-command-bad-returnType",
+		URL:         "core/command",
+		Method:      "POST",
+		Body:        `command=version&returnType=POTATO`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusInternalServerError,
+		Expected: `{
+	"error": "Unknown returnType \"POTATO\"",
+	"input": {
+		"command": "version",
+		"returnType": "POTATO"
+	},
+	"path": "core/command",
+	"status": 500
+}
+`,
+	}, {
+		Name:        "core-command-stream",
+		URL:         "core/command",
+		Method:      "POST",
+		Body:        `command=version&returnType=STREAM`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusOK,
+		Expected: fmt.Sprintf(`rclone %s
+{}
+`, fs.Version),
+	}, {
+		Name:        "core-command-stream-error",
+		URL:         "core/command",
+		Method:      "POST",
+		Body:        `command=unknown_command&returnType=STREAM`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusOK,
+		Expected: fmt.Sprintf(`rclone %s
+Unknown command
+{
+	"error": "exit status 1",
+	"input": {
+		"command": "unknown_command",
+		"returnType": "STREAM"
+	},
+	"path": "core/command",
+	"status": 500
+}
+`, fs.Version),
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	opt.NoAuth = true
+	testServer(t, tests, &opt)
+}
+
 func TestMethods(t *testing.T) {
 	tests := []testRun{{
 		Name:     "options",

fs/rc/webgui/plugins.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/rc/rcflags"
+	"github.com/rclone/rclone/lib/errors"
 )
 
 // PackageJSON is the structure of package.json of a plugin
@@ -62,6 +64,8 @@ var (
 	PluginsPath              string
 	pluginsConfigPath        string
 	availablePluginsJSONPath = "availablePlugins.json"
+	initSuccess              = false
+	initMutex                = &sync.Mutex{}
 )
 
 func init() {
@@ -69,11 +73,6 @@ func init() {
 	PluginsPath = filepath.Join(cachePath, "plugins")
 	pluginsConfigPath = filepath.Join(PluginsPath, "config")
 
-	loadedPlugins = newPlugins(availablePluginsJSONPath)
-	err := loadedPlugins.readFromFile()
-	if err != nil {
-		fs.Errorf(nil, "error reading available plugins: %v", err)
-	}
 }
 
 // Plugins represents the structure how plugins are saved onto disk
@@ -90,9 +89,25 @@ func newPlugins(fileName string) *Plugins {
 	return &p
 }
 
+func initPluginsOrError() error {
+	if !rcflags.Opt.WebUI {
+		return errors.New("WebUI needs to be enabled for plugins to work")
+	}
+	initMutex.Lock()
+	defer initMutex.Unlock()
+	if !initSuccess {
+		loadedPlugins = newPlugins(availablePluginsJSONPath)
+		err := loadedPlugins.readFromFile()
+		if err != nil {
+			fs.Errorf(nil, "error reading available plugins: %v", err)
+		}
+		initSuccess = true
+	}
+
+	return nil
+}
+
 func (p *Plugins) readFromFile() (err error) {
-	//p.mutex.Lock()
-	//defer p.mutex.Unlock()
 	err = CreatePathIfNotExist(pluginsConfigPath)
 	if err != nil {
 		return err
@@ -169,8 +184,6 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [
 }
 
 func (p *Plugins) writeToFile() (err error) {
-	//p.mutex.Lock()
-	//defer p.mutex.Unlock()
 	availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
 
 	file, err := json.MarshalIndent(p, "", " ")
@@ -290,6 +303,10 @@ var referrerPathReg = regexp.MustCompile("^(https?):\\/\\/(.+):([0-9]+)?\\/(.*)\
 // sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when
 // the referrer in http.Request is set
 func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path string) (ok bool) {
+	err := initPluginsOrError()
+	if err != nil {
+		return false
+	}
 	referrer := r.Referer()
 	referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer)
 

fs/rc/webgui/rc.go

@@ -30,6 +30,10 @@ Eg
 }
 
 func rcListTestPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	return rc.Params{
 		"loadedTestPlugins": filterPlugins(loadedPlugins, func(json *PackageJSON) bool { return json.isTesting() }),
 	}, nil
@@ -54,6 +58,10 @@ Eg
 	})
 }
 func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	name, err := in.GetString("name")
 	if err != nil {
 		return nil, err
@@ -85,6 +93,10 @@ Eg
 }
 
 func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	pluginURL, err := in.GetString("url")
 	if err != nil {
 		return nil, err
@@ -192,6 +204,10 @@ Eg
 }
 
 func rcGetPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	err = loadedPlugins.readFromFile()
 	if err != nil {
 		return nil, err
@@ -222,6 +238,10 @@ Eg
 }
 
 func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	name, err := in.GetString("name")
 	if err != nil {
 		return nil, err
@@ -260,6 +280,10 @@ Eg
 }
 
 func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	err = initPluginsOrError()
+	if err != nil {
+		return nil, err
+	}
 	handlesType, err := in.GetString("type")
 	if err != nil {
 		handlesType = ""

fs/rc/webgui/rc_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -18,6 +19,10 @@ const testPluginAuthor = "rclone"
 const testPluginKey = testPluginAuthor + "/" + testPluginName
 const testPluginURL = "https://github.com/" + testPluginAuthor + "/" + testPluginName + "/"
 
+func init() {
+	rcflags.Opt.WebUI = true
+}
+
 func setCacheDir(t *testing.T) string {
 	cacheDir, err := ioutil.TempDir("", "rclone-cache-dir")
 	assert.Nil(t, err)

fs/version.go

@@ -1,4 +1,4 @@
 package fs
 
 // Version of rclone
-var Version = "v1.53.2-DEV"
+var Version = "v1.53.4-DEV"

lib/random/random.go

@@ -2,8 +2,10 @@
 package random
 
 import (
+	cryptorand "crypto/rand"
 	"encoding/base64"
-	"math/rand"
+	"encoding/binary"
+	mathrand "math/rand"
 
 	"github.com/pkg/errors"
 )
@@ -23,7 +25,7 @@ func String(n int) string {
 	for i := range out {
 		source := pattern[p]
 		p = (p + 1) % len(pattern)
-		out[i] = source[rand.Intn(len(source))]
+		out[i] = source[mathrand.Intn(len(source))]
 	}
 	return string(out)
 }
@@ -41,7 +43,7 @@ func Password(bits int) (password string, err error) {
 		bytes++
 	}
 	var pw = make([]byte, bytes)
-	n, err := rand.Read(pw)
+	n, err := cryptorand.Read(pw)
 	if err != nil {
 		return "", errors.Wrap(err, "password read failed")
 	}
@@ -51,3 +53,19 @@ func Password(bits int) (password string, err error) {
 	password = base64.RawURLEncoding.EncodeToString(pw)
 	return password, nil
 }
+
+// Seed the global math/rand with crypto strong data
+//
+// This doesn't make it OK to use math/rand in crypto sensitive
+// environments - don't do that! However it does help to mitigate the
+// problem if that happens accidentally. This would have helped with
+// CVE-2020-28924 - #4783
+func Seed() error {
+	var seed int64
+	err := binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
+	if err != nil {
+		return errors.Wrap(err, "failed to read random seed")
+	}
+	mathrand.Seed(seed)
+	return nil
+}

lib/random/random_test.go

@@ -1,6 +1,7 @@
 package random
 
 import (
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -48,3 +49,16 @@ func TestPasswordDuplicates(t *testing.T) {
 		seen[s] = true
 	}
 }
+
+func TestSeed(t *testing.T) {
+	// seed 100 times and check the first random number doesn't repeat
+	// This test could fail with a probability of ~ 10**-15
+	const n = 100
+	var seen = map[int64]bool{}
+	for i := 0; i < n; i++ {
+		assert.NoError(t, Seed())
+		first := rand.Int63()
+		assert.False(t, seen[first])
+		seen[first] = true
+	}
+}

rclone.1

@@ -1,7 +1,7 @@
 .\"t
 .\" Automatically generated by Pandoc 2.5
 .\"
-.TH "rclone" "1" "Oct 26, 2020" "User Manual" ""
+.TH "rclone" "1" "Nov 19, 2020" "User Manual" ""
 .hy
 .SH Rclone syncs your files to cloud storage
 .PP
@@ -14154,7 +14154,7 @@ These flags are available for every command.
       \-\-use\-json\-log                         Use json log format.
       \-\-use\-mmap                             Use mmap allocator (see docs).
       \-\-use\-server\-modtime                   Use server modified time instead of object metadata
-      \-\-user\-agent string                    Set the user\-agent to a specified string. The default is rclone/ version (default \[dq]rclone/v1.53.2\[dq])
+      \-\-user\-agent string                    Set the user\-agent to a specified string. The default is rclone/ version (default \[dq]rclone/v1.53.3\[dq])
   \-v, \-\-verbose count                        Print lots more stuff (repeat for more)
 \f[R]
 .fi
@@ -35088,6 +35088,39 @@ Options:
 .IP \[bu] 2
 \[dq]error\[dq]: return an error based on option value
 .SH Changelog
+.SS v1.53.3 \- 2020\-11\-19
+.PP
+See commits (https://github.com/rclone/rclone/compare/v1.53.2...v1.53.3)
+.IP \[bu] 2
+Bug Fixes
+.RS 2
+.IP \[bu] 2
+random: Fix incorrect use of math/rand instead of crypto/rand
+CVE\-2020\-28924 (Nick Craig\-Wood)
+.RS 2
+.IP \[bu] 2
+Passwords you have generated with \f[C]rclone config\f[R] may be
+insecure
+.IP \[bu] 2
+See issue #4783 (https://github.com/rclone/rclone/issues/4783) for more
+details and a checking tool
+.RE
+.IP \[bu] 2
+random: Seed math/rand in one place with crypto strong seed (Nick
+Craig\-Wood)
+.RE
+.IP \[bu] 2
+VFS
+.RS 2
+.IP \[bu] 2
+Fix vfs/refresh calls with fs= parameter (Nick Craig\-Wood)
+.RE
+.IP \[bu] 2
+Sharefile
+.RS 2
+.IP \[bu] 2
+Fix backend due to API swapping integers for strings (Nick Craig\-Wood)
+.RE
 .SS v1.53.2 \- 2020\-10\-26
 .PP
 See commits (https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)

vfs/rc.go

@@ -23,6 +23,8 @@ must be supplied.`
 //
 // If "fs" is not set and there is one and only one VFS in the active
 // cache then it returns it. This is for backwards compatibility.
+//
+// This deletes the "fs" parameter from in if it is valid
 func getVFS(in rc.Params) (vfs *VFS, err error) {
 	fsString, err := in.GetString("fs")
 	if rc.IsErrParamNotFound(err) {
@@ -46,6 +48,7 @@ func getVFS(in rc.Params) (vfs *VFS, err error) {
 	} else if len(activeVFS) > 1 {
 		return nil, errors.Errorf("more than one VFS active with name %q", fsString)
 	}
+	delete(in, "fs") // delete the fs parameter
 	return activeVFS[0], nil
 }
 

vfs/rc_test.go

@@ -57,6 +57,7 @@ func TestRcGetVFS(t *testing.T) {
 	assert.Contains(t, err.Error(), "more than one VFS active - need")
 	assert.Nil(t, vfs)
 
+	inPresent = rc.Params{"fs": fs.ConfigString(r.Fremote)}
 	vfs, err = getVFS(inPresent)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "more than one VFS active with name")
@@ -67,7 +68,8 @@ func TestRcForget(t *testing.T) {
 	r, vfs, cleanup, call := rcNewRun(t, "vfs/forget")
 	defer cleanup()
 	_, _ = r, vfs
-	out, err := call.Fn(context.Background(), nil)
+	in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
+	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	assert.Equal(t, rc.Params{
 		"forgotten": []string{},
@@ -79,7 +81,8 @@ func TestRcRefresh(t *testing.T) {
 	r, vfs, cleanup, call := rcNewRun(t, "vfs/refresh")
 	defer cleanup()
 	_, _ = r, vfs
-	out, err := call.Fn(context.Background(), nil)
+	in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
+	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	assert.Equal(t, rc.Params{
 		"result": map[string]string{

vfs/vfscache/cache.go

@@ -80,9 +80,14 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir
 		}
 		fRoot = strings.Replace(fRoot, ":", "", -1)
 	}
-	root := file.UNCPath(filepath.Join(config.CacheDir, "vfs", fremote.Name(), fRoot))
+	cacheDir := config.CacheDir
+	cacheDir, err := filepath.Abs(cacheDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make --cache-dir absolute")
+	}
+	root := file.UNCPath(filepath.Join(cacheDir, "vfs", fremote.Name(), fRoot))
 	fs.Debugf(nil, "vfs cache: root is %q", root)
-	metaRoot := file.UNCPath(filepath.Join(config.CacheDir, "vfsMeta", fremote.Name(), fRoot))
+	metaRoot := file.UNCPath(filepath.Join(cacheDir, "vfsMeta", fremote.Name(), fRoot))
 	fs.Debugf(nil, "vfs cache: metadata root is %q", root)
 
 	fcache, err := fscache.Get(root)