vfs: check destination on rename and delete if it exists first FIXME needs tests

Some backends (eg onedrive) will throw an error if we rename over an
existing file and some (drive) will create a duplicate.

This patch checks to see if the destination exists and removes it or
returns an error as appropriate.

Fixes #4293

.github/workflows/build.yml

@@ -19,13 +19,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.11', 'go1.12', 'go1.13']
+        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'race', 'go1.11', 'go1.12', 'go1.13']
 
         include:
           - job_name: linux
             os: ubuntu-latest
             go: '1.14.x'
-            modules: 'off'
+            modules: 'on'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -35,7 +35,7 @@ jobs:
           - job_name: mac
             os: macOS-latest
             go: '1.14.x'
-            modules: 'off'
+            modules: 'on'
             gotags: ''  # cmount doesn't work on osx travis for some reason
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -45,7 +45,7 @@ jobs:
           - job_name: windows_amd64
             os: windows-latest
             go: '1.14.x'
-            modules: 'off'
+            modules: 'on'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             quicktest: true
@@ -55,7 +55,7 @@ jobs:
           - job_name: windows_386
             os: windows-latest
             go: '1.14.x'
-            modules: 'off'
+            modules: 'on'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -66,12 +66,12 @@ jobs:
           - job_name: other_os
             os: ubuntu-latest
             go: '1.14.x'
-            modules: 'off'
+            modules: 'on'
             build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: modules_race
+          - job_name: race
             os: ubuntu-latest
             go: '1.14.x'
             modules: 'on'
@@ -81,19 +81,19 @@ jobs:
           - job_name: go1.11
             os: ubuntu-latest
             go: '1.11.x'
-            modules: 'off'
+            modules: 'on'
             quicktest: true
 
           - job_name: go1.12
             os: ubuntu-latest
             go: '1.12.x'
-            modules: 'off'
+            modules: 'on'
             quicktest: true
 
           - job_name: go1.13
             os: ubuntu-latest
             go: '1.13.x'
-            modules: 'off'
+            modules: 'on'
             quicktest: true
 
     name: ${{ matrix.job_name }}
@@ -229,12 +229,12 @@ jobs:
         run: |
           docker pull billziss/xgo-cgofuse
           GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
-          xgo \
-              -image=billziss/xgo-cgofuse \
-              -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-              -tags cmount \
-              -dest build \
-              .
+          # xgo \
+          #     -image=billziss/xgo-cgofuse \
+          #     -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+          #     -tags cmount \
+          #     -dest build \
+          #     .
           xgo \
               -image=billziss/xgo-cgofuse \
               -targets=android/*,ios/* \
@@ -244,7 +244,7 @@ jobs:
       - name: Build rclone
         run: |
           docker pull golang
-          docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
+          docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=mod -v
 
       - name: Upload artifacts
         run: |

CONTRIBUTING.md

@@ -186,7 +186,6 @@ with modules beneath.
     * pacer - retries with backoff and paces operations
     * readers - a selection of useful io.Readers
     * rest - a thin abstraction over net/http for REST
-  * vendor - 3rd party code managed by `go mod`
   * vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
 ## Writing Documentation ##
@@ -266,41 +265,25 @@ rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-**NB** you must be using go1.11 or above to add a dependency to
-rclone.  Rclone will still build with older versions of go, but we use
-the `go mod` command for dependencies which is only in go1.11 and
-above.
-
-rclone can be built with modules outside of the GOPATH, but for
-backwards compatibility with older go versions, rclone also maintains
-a `vendor` directory with all the external code rclone needs for
-building.
-
-The `vendor` directory is entirely managed by the `go mod` tool, do
-not add things manually.
+rclone can be built with modules outside of the GOPATH
 
 To add a dependency `github.com/ncw/new_dependency` see the
-instructions below.  These will fetch the dependency, add it to
-`go.mod` and `go.sum` and vendor it for older go versions.
+instructions below.  These will fetch the dependency and add it to
+`go.mod` and `go.sum`.
 
     GO111MODULE=on go get github.com/ncw/new_dependency
-    GO111MODULE=on go mod vendor
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
 
-Please check in the changes generated by `go mod` including the
-`vendor` directory and `go.mod` and `go.sum` in a single commit
-separate from any other code changes with the title "vendor: add
-github.com/ncw/new_dependency".  Remember to `git add` any new files
-in `vendor`.
+Please check in the changes generated by `go mod` including `go.mod`
+and `go.sum` in the same commit as your other changes.
 
 ## Updating a dependency ##
 
 If you need to update a dependency then run
 
     GO111MODULE=on go get -u github.com/pkg/errors
-    GO111MODULE=on go mod vendor
 
 Check in a single commit as above.
 

Makefile

@@ -27,7 +27,7 @@ ifndef RELEASE_TAG
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
-GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+GO_FILES := $(shell go list ./... )
 ifdef BETA_SUBDIR
 	BETA_SUBDIR := /$(BETA_SUBDIR)
 endif
@@ -100,15 +100,19 @@ release_dep_windows:
 	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
 
 # Update dependencies
+showupdates:
+	@echo "*** Direct dependencies that could be updated ***"
+	@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
+
+# Update direct and indirect dependencies and test dependencies
 update:
-	GO111MODULE=on go get -u ./...
+	GO111MODULE=on go get -u -t ./...
+	-#GO111MODULE=on go get -d $(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
 	GO111MODULE=on go mod tidy
-	GO111MODULE=on go mod vendor
 
 # Tidy the module dependencies
 tidy:
 	GO111MODULE=on go mod tidy
-	GO111MODULE=on go mod vendor
 
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 

RELEASE.md

@@ -33,7 +33,7 @@ This file describes how to make the various kinds of releases
   * make startdev
   * # announce with forum post, twitter post, patreon post
 
-Early in the next release cycle update the vendored dependencies
+Early in the next release cycle update the dependencies
 
   * Review any pinned packages in go.mod and remove if possible
   * make update
@@ -53,7 +53,6 @@ Can be fixed with
 
     * GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
     * GO111MODULE=on go mod tidy
-    * GO111MODULE=on go mod vendor
  
 
 ## Making a point release

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.13
 
 package azureblob
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.13
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.13
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris
+// +build plan9 solaris !go1.13
 
 package azureblob

backend/drive/drive.go

@@ -57,6 +57,7 @@ const (
 	rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
 	driveFolderType             = "application/vnd.google-apps.folder"
 	shortcutMimeType            = "application/vnd.google-apps.shortcut"
+	shortcutMimeTypeDangling    = "application/vnd.google-apps.shortcut.dangling" // synthetic mime type for internal use
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
@@ -1356,6 +1357,10 @@ func (f *Fs) newObjectWithExportInfo(
 		// and not from a listing. This is unlikely.
 		fs.Debugf(remote, "Ignoring shortcut as skip shortcuts is set")
 		return nil, fs.ErrorObjectNotFound
+	case info.MimeType == shortcutMimeTypeDangling:
+		// Pretend a dangling shortcut is a regular object
+		// It will error if used, but appear in listings so it can be deleted
+		return f.newRegularObject(remote, info), nil
 	case info.Md5Checksum != "" || info.Size > 0:
 		// If item has MD5 sum or a length it is a file stored on drive
 		return f.newRegularObject(remote, info), nil
@@ -1980,6 +1985,12 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 	}
 	newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
+		if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
+			// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
+			fs.Logf(nil, "Dangling shortcut %q detected", item.Name)
+			item.MimeType = shortcutMimeTypeDangling
+			return item, nil
+		}
 		return nil, errors.Wrap(err, "failed to resolve shortcut")
 	}
 	// make sure we use the Name, Parents and Trashed from the original item
@@ -3223,6 +3234,9 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.mimeType == shortcutMimeTypeDangling {
+		return nil, errors.New("can't read dangling shortcut")
+	}
 	if o.v2Download {
 		var v2File *drive_v2.File
 		err = o.fs.pacer.Call(func() (bool, error) {

backend/ftp/ftp.go

@@ -8,6 +8,8 @@ import (
 	"net/textproto"
 	"os"
 	"path"
+	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -160,6 +162,43 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
+// Enable debugging output
+type debugLog struct {
+	mu   sync.Mutex
+	auth bool
+}
+
+// Write writes len(p) bytes from p to the underlying data stream. It returns
+// the number of bytes written from p (0 <= n <= len(p)) and any error
+// encountered that caused the write to stop early. Write must return a non-nil
+// error if it returns n < len(p). Write must not modify the slice data, even
+// temporarily.
+//
+// Implementations must not retain p.
+//
+// This writes debug info to the log
+func (dl *debugLog) Write(p []byte) (n int, err error) {
+	dl.mu.Lock()
+	defer dl.mu.Unlock()
+	_, file, _, ok := runtime.Caller(1)
+	direction := "FTP Rx"
+	if ok && strings.Contains(file, "multi") {
+		direction = "FTP Tx"
+	}
+	lines := strings.Split(string(p), "\r\n")
+	if lines[len(lines)-1] == "" {
+		lines = lines[:len(lines)-1]
+	}
+	for _, line := range lines {
+		if !dl.auth && strings.HasPrefix(line, "PASS") {
+			fs.Debugf(direction, "PASS *****")
+			continue
+		}
+		fs.Debugf(direction, "%q", line)
+	}
+	return len(p), nil
+}
+
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
@@ -183,6 +222,9 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
 	}
+	if fs.Config.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
+		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: fs.Config.Dump&fs.DumpAuth != 0}))
+	}
 	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
 		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)

backend/googlecloudstorage/googlecloudstorage.go

@@ -79,7 +79,8 @@ func init() {
 		Config: func(name string, m configmap.Mapper) {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
-			if saFile != "" || saCreds != "" {
+			anonymous, _ := m.Get("anonymous")
+			if saFile != "" || saCreds != "" || anonymous == "true" {
 				return
 			}
 			err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
@@ -103,6 +104,10 @@ func init() {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
 			Hide: fs.OptionHideBoth,
+		}, {
+			Name:    "anonymous",
+			Help:    "Access public buckets and objects without credentials\nSet to 'true' if you just want to download files and don't configure credentials.",
+			Default: false,
 		}, {
 			Name: "object_acl",
 			Help: "Access Control List for new objects.",
@@ -265,6 +270,7 @@ type Options struct {
 	ProjectNumber             string               `config:"project_number"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
+	Anonymous                 bool                 `config:"anonymous"`
 	ObjectACL                 string               `config:"object_acl"`
 	BucketACL                 string               `config:"bucket_acl"`
 	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`
@@ -411,7 +417,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		opt.ServiceAccountCredentials = string(loadedCreds)
 	}
-	if opt.ServiceAccountCredentials != "" {
+	if opt.Anonymous {
+		oAuthClient = &http.Client{}
+	} else if opt.ServiceAccountCredentials != "" {
 		oAuthClient, err = getServiceAccountClient([]byte(opt.ServiceAccountCredentials))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")

backend/local/local.go

@@ -10,7 +10,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -183,20 +182,22 @@ type Fs struct {
 	warned      map[string]struct{} // whether we have warned about this string
 
 	// do os.Lstat or os.Stat
-	lstat          func(name string) (os.FileInfo, error)
-	objectHashesMu sync.Mutex // global lock for Object.hashes
+	lstat        func(name string) (os.FileInfo, error)
+	objectMetaMu sync.RWMutex // global lock for Object metadata
 }
 
 // Object represents a local filesystem object
 type Object struct {
-	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path (encoded path)
-	path           string // The local path (OS path)
-	size           int64  // file metadata - always present
-	mode           os.FileMode
-	modTime        time.Time
-	hashes         map[hash.Type]string // Hashes
-	translatedLink bool                 // Is this object a translated link
+	fs     *Fs    // The Fs this object is part of
+	remote string // The remote path (encoded path)
+	path   string // The local path (OS path)
+	// When using these items the fs.objectMetaMu must be held
+	size    int64 // file metadata - always present
+	mode    os.FileMode
+	modTime time.Time
+	hashes  map[hash.Type]string // Hashes
+	// these are read only and don't need the mutex held
+	translatedLink bool // Is this object a translated link
 }
 
 // ------------------------------------------------------------
@@ -231,6 +232,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
 		IsLocal:                 true,
+		SlowHash:                true,
 	}).Fill(f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -632,6 +634,9 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	// Temporary Object under construction
 	dstObj := f.newObject(remote)
+	dstObj.fs.objectMetaMu.RLock()
+	dstObjMode := dstObj.mode
+	dstObj.fs.objectMetaMu.RUnlock()
 
 	// Check it is a file if it exists
 	err := dstObj.lstat()
@@ -639,7 +644,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.fs.isRegular(dstObj.mode) {
+	} else if !dstObj.fs.isRegular(dstObjMode) {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -793,8 +798,10 @@ func (o *Object) Remote() string {
 // Hash returns the requested hash of a file as a lowercase hex string
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	// Check that the underlying file hasn't changed
+	o.fs.objectMetaMu.RLock()
 	oldtime := o.modTime
 	oldsize := o.size
+	o.fs.objectMetaMu.RUnlock()
 	err := o.lstat()
 	var changed bool
 	if err != nil {
@@ -806,15 +813,16 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", errors.Wrap(err, "hash: failed to stat")
 		}
 	} else {
+		o.fs.objectMetaMu.RLock()
 		changed = !o.modTime.Equal(oldtime) || oldsize != o.size
+		o.fs.objectMetaMu.RUnlock()
 	}
 
-	o.fs.objectHashesMu.Lock()
-	hashes := o.hashes
+	o.fs.objectMetaMu.RLock()
 	hashValue, hashFound := o.hashes[r]
-	o.fs.objectHashesMu.Unlock()
+	o.fs.objectMetaMu.RUnlock()
 
-	if changed || hashes == nil || !hashFound {
+	if changed || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -833,6 +841,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
+		var hashes map[hash.Type]string
 		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
@@ -842,24 +851,28 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", errors.Wrap(closeErr, "hash: failed to close")
 		}
 		hashValue = hashes[r]
-		o.fs.objectHashesMu.Lock()
+		o.fs.objectMetaMu.Lock()
 		if o.hashes == nil {
 			o.hashes = hashes
 		} else {
 			o.hashes[r] = hashValue
 		}
-		o.fs.objectHashesMu.Unlock()
+		o.fs.objectMetaMu.Unlock()
 	}
 	return hashValue, nil
 }
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
+	o.fs.objectMetaMu.RLock()
+	defer o.fs.objectMetaMu.RUnlock()
 	return o.size
 }
 
 // ModTime returns the modification time of the object
 func (o *Object) ModTime(ctx context.Context) time.Time {
+	o.fs.objectMetaMu.RLock()
+	defer o.fs.objectMetaMu.RUnlock()
 	return o.modTime
 }
 
@@ -880,7 +893,9 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 
 // Storable returns a boolean showing if this object is storable
 func (o *Object) Storable() bool {
+	o.fs.objectMetaMu.RLock()
 	mode := o.mode
+	o.fs.objectMetaMu.RUnlock()
 	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
@@ -913,11 +928,15 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 		if err != nil {
 			return 0, errors.Wrap(err, "can't read status of source file while transferring")
 		}
-		if file.o.size != fi.Size() {
-			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()))
+		file.o.fs.objectMetaMu.RLock()
+		oldtime := file.o.modTime
+		oldsize := file.o.size
+		file.o.fs.objectMetaMu.RUnlock()
+		if oldsize != fi.Size() {
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", oldsize, fi.Size()))
 		}
-		if !file.o.modTime.Equal(fi.ModTime()) {
-			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()))
+		if !oldtime.Equal(fi.ModTime()) {
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", oldtime, fi.ModTime()))
 		}
 	}
 
@@ -934,9 +953,9 @@ func (file *localOpenFile) Close() (err error) {
 	err = file.in.Close()
 	if err == nil {
 		if file.hash.Size() == file.o.Size() {
-			file.o.fs.objectHashesMu.Lock()
+			file.o.fs.objectMetaMu.Lock()
 			file.o.hashes = file.hash.Sums()
-			file.o.fs.objectHashesMu.Unlock()
+			file.o.fs.objectMetaMu.Unlock()
 		}
 	}
 	return err
@@ -961,7 +980,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		case *fs.SeekOption:
 			offset = x.Offset
 		case *fs.RangeOption:
-			offset, limit = x.Decode(o.size)
+			offset, limit = x.Decode(o.Size())
 		case *fs.HashesOption:
 			if x.Hashes.Count() > 0 {
 				hasher, err = hash.NewMultiHasherTypes(x.Hashes)
@@ -1119,9 +1138,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// All successful so update the hashes
 	if hasher != nil {
-		o.fs.objectHashesMu.Lock()
+		o.fs.objectMetaMu.Lock()
 		o.hashes = hasher.Sums()
-		o.fs.objectHashesMu.Unlock()
+		o.fs.objectMetaMu.Unlock()
 	}
 
 	// Set the mtime
@@ -1183,17 +1202,11 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
 		return
 	}
-	// Don't overwrite the info if we don't need to
-	// this avoids upsetting the race detector
-	if o.size != info.Size() {
-		o.size = info.Size()
-	}
-	if !o.modTime.Equal(info.ModTime()) {
-		o.modTime = info.ModTime()
-	}
-	if o.mode != info.Mode() {
-		o.mode = info.Mode()
-	}
+	o.fs.objectMetaMu.Lock()
+	o.size = info.Size()
+	o.modTime = info.ModTime()
+	o.mode = info.Mode()
+	o.fs.objectMetaMu.Unlock()
 }
 
 // Stat an Object into info
@@ -1225,7 +1238,7 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 
 		if !noUNC {
 			// Convert to UNC
-			s = uncPath(s)
+			s = file.UNCPath(s)
 		}
 		return s
 	}
@@ -1239,28 +1252,6 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 	return s
 }
 
-// Pattern to match a windows absolute path: "c:\" and similar
-var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
-
-// uncPath converts an absolute Windows path
-// to a UNC long path.
-func uncPath(l string) string {
-	// If prefix is "\\", we already have a UNC path or server.
-	if strings.HasPrefix(l, `\\`) {
-		// If already long path, just keep it
-		if strings.HasPrefix(l, `\\?\`) {
-			return l
-		}
-
-		// Trim "\\" from path and add UNC prefix.
-		return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
-	}
-	if isAbsWinDrive.MatchString(l) {
-		return `\\?\` + l
-	}
-	return l
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs             = &Fs{}

backend/local/tests_test.go

@@ -5,49 +5,6 @@ import (
 	"testing"
 )
 
-var uncTestPaths = []string{
-	`C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`C:\Windows\Folder`,
-	`\\?\C:\Windows\Folder`,
-	`\\?\UNC\server\share\Desktop`,
-	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\server\share\Desktop`,
-	`\\?\UNC\\share\folder\Desktop`,
-	`\\server\share`,
-}
-
-var uncTestPathsResults = []string{
-	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`\\?\C:\Windows\Folder`,
-	`\\?\C:\Windows\Folder`,
-	`\\?\UNC\server\share\Desktop`,
-	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
-	`\\?\UNC\server\share\Desktop`,
-	`\\?\UNC\\share\folder\Desktop`,
-	`\\?\UNC\server\share`,
-}
-
-// Test that UNC paths are converted.
-func TestUncPaths(t *testing.T) {
-	for i, p := range uncTestPaths {
-		unc := uncPath(p)
-		if unc != uncTestPathsResults[i] {
-			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
-		}
-		// Test we don't add more.
-		unc = uncPath(unc)
-		if unc != uncTestPathsResults[i] {
-			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
-		}
-	}
-}
-
 // Test Windows character replacements
 var testsWindows = [][2]string{
 	{`c:\temp`, `c:\temp`},

backend/onedrive/onedrive.go

@@ -1107,9 +1107,10 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	id, dstDriveID, _ := parseNormalizedID(directoryID)
 	_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
 
-	if dstDriveID != srcObjDriveID {
+	if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
+		fs.Debugf(f, "Can't move files between drives (%q != %q)", dstDriveID, srcObjDriveID)
 		return nil, fs.ErrorCantMove
 	}
 
@@ -1168,9 +1169,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
-	if dstDriveID != srcDriveID {
+	if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
+		fs.Debugf(f, "Can't move directories between drives (%q != %q)", dstDriveID, srcDriveID)
 		return fs.ErrorCantDirMove
 	}
 
@@ -1787,6 +1789,17 @@ func parseNormalizedID(ID string) (string, string, string) {
 	return ID, "", ""
 }
 
+// Returns the canonical form of the driveID
+func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
+	if driveID == "" {
+		canonicalDriveID = f.opt.DriveID
+	} else {
+		canonicalDriveID = driveID
+	}
+	canonicalDriveID = strings.ToLower(canonicalDriveID)
+	return canonicalDriveID
+}
+
 // getRelativePathInsideBase checks if `target` is inside `base`. If so, it
 // returns a relative path for `target` based on `base` and a boolean `true`.
 // Otherwise returns "", false.

backend/pcloud/pcloud.go

@@ -42,7 +42,7 @@ const (
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
-	rootURL                     = "https://api.pcloud.com"
+	defaultHostname             = "api.pcloud.com"
 )
 
 // Globals
@@ -51,8 +51,8 @@ var (
 	oauthConfig = &oauth2.Config{
 		Scopes: nil,
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://my.pcloud.com/oauth2/authorize",
-			TokenURL: "https://api.pcloud.com/oauth2_token",
+			AuthURL: "https://my.pcloud.com/oauth2/authorize",
+			// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
 		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
@@ -60,17 +60,45 @@ var (
 	}
 )
 
+// Update the TokenURL with the actual hostname
+func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
+	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
+}
+
 // Register with Fs
 func init() {
+	updateTokenURL(oauthConfig, defaultHostname)
 	fs.Register(&fs.RegInfo{
 		Name:        "pcloud",
 		Description: "Pcloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
+			optc := new(Options)
+			err := configstruct.Set(m, optc)
+			if err != nil {
+				fs.Errorf(nil, "Failed to read config: %v", err)
+			}
+			updateTokenURL(oauthConfig, optc.Hostname)
+			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
+				if auth == nil || auth.Form == nil {
+					return errors.New("form not found in response")
+				}
+				hostname := auth.Form.Get("hostname")
+				if hostname == "" {
+					hostname = defaultHostname
+				}
+				// Save the hostname in the config
+				m.Set("hostname", hostname)
+				// Update the token URL
+				updateTokenURL(oauthConfig, hostname)
+				fs.Debugf(nil, "pcloud: got hostname %q", hostname)
+				return nil
+			}
 			opt := oauthutil.Options{
+				CheckAuth:    checkAuth,
 				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
 			}
-			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
+			err = oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -96,6 +124,13 @@ func init() {
 			Help:     "Fill in for rclone to use a non root folder as its starting point.",
 			Default:  "d0",
 			Advanced: true,
+		}, {
+			Name: "hostname",
+			Help: `Hostname to connect to.
+
+This is normally set when rclone initially does the oauth connection.`,
+			Default:  defaultHostname,
+			Advanced: true,
 		}},
 	})
 }
@@ -104,6 +139,7 @@ func init() {
 type Options struct {
 	Enc          encoder.MultiEncoder `config:"encoding"`
 	RootFolderID string               `config:"root_folder_id"`
+	Hostname     string               `config:"hostname"`
 }
 
 // Fs represents a remote pcloud
@@ -253,12 +289,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure Pcloud")
 	}
+	updateTokenURL(oauthConfig, opt.Hostname)
 
 	f := &Fs{
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
+		srv:   rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),
 		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{

backend/qingstor/qingstor.go

@@ -356,6 +356,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:     true,
 		BucketBased:       true,
 		BucketBasedRootOK: true,
+		SlowModTime:       true,
 	}).Fill(f)
 
 	if f.rootBucket != "" && f.rootDirectory != "" {

backend/s3/s3.go

@@ -1188,9 +1188,6 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		return nil, nil, errors.New("secret_access_key not found")
 	}
 
-	if opt.Region == "" && opt.Endpoint == "" {
-		opt.Endpoint = "https://s3.amazonaws.com/"
-	}
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
@@ -1334,6 +1331,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		BucketBasedRootOK: true,
 		SetTier:           true,
 		GetTier:           true,
+		SlowModTime:       true,
 	}).Fill(f)
 	if f.rootBucket != "" && f.rootDirectory != "" {
 		// Check to see if the object exists

backend/sftp/sftp.go

@@ -193,6 +193,7 @@ type Options struct {
 type Fs struct {
 	name         string
 	root         string
+	absRoot      string
 	opt          Options          // parsed options
 	m            configmap.Mapper // config
 	features     *fs.Features     // optional features
@@ -491,6 +492,7 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
 	f := &Fs{
 		name:      name,
 		root:      root,
+		absRoot:   root,
 		opt:       *opt,
 		m:         m,
 		config:    sshConfig,
@@ -500,17 +502,27 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
+		SlowHash:                true,
 	}).Fill(f)
 	// Make a connection and pool it to return errors early
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "NewFs")
 	}
+	cwd, err := c.sftpClient.Getwd()
 	f.putSftpConnection(&c, nil)
+	if err != nil {
+		fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
+	} else if !path.IsAbs(f.root) {
+		f.absRoot = path.Join(cwd, f.root)
+		fs.Debugf(f, "Using absolute root directory %q", f.absRoot)
+	}
 	if root != "" {
 		// Check to see if the root actually an existing file
+		oldAbsRoot := f.absRoot
 		remote := path.Base(root)
 		f.root = path.Dir(root)
+		f.absRoot = path.Dir(f.absRoot)
 		if f.root == "." {
 			f.root = ""
 		}
@@ -519,6 +531,7 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
 			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
 				// File doesn't exist so return old f
 				f.root = root
+				f.absRoot = oldAbsRoot
 				return f, nil
 			}
 			return nil, err
@@ -601,7 +614,7 @@ func (f *Fs) dirExists(dir string) (bool, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	root := path.Join(f.root, dir)
+	root := path.Join(f.absRoot, dir)
 	ok, err := f.dirExists(root)
 	if err != nil {
 		return nil, errors.Wrap(err, "List failed")
@@ -682,7 +695,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // directories above that
 func (f *Fs) mkParentDir(remote string) error {
 	parent := path.Dir(remote)
-	return f.mkdir(path.Join(f.root, parent))
+	return f.mkdir(path.Join(f.absRoot, parent))
 }
 
 // mkdir makes the directory and parents using native paths
@@ -718,7 +731,7 @@ func (f *Fs) mkdir(dirPath string) error {
 
 // Mkdir makes the root directory of the Fs object
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	root := path.Join(f.root, dir)
+	root := path.Join(f.absRoot, dir)
 	return f.mkdir(root)
 }
 
@@ -734,7 +747,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return fs.ErrorDirectoryNotEmpty
 	}
 	// Remove the directory
-	root := path.Join(f.root, dir)
+	root := path.Join(f.absRoot, dir)
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return errors.Wrap(err, "Rmdir")
@@ -761,7 +774,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	err = c.sftpClient.Rename(
 		srcObj.path(),
-		path.Join(f.root, remote),
+		path.Join(f.absRoot, remote),
 	)
 	f.putSftpConnection(&c, err)
 	if err != nil {
@@ -788,8 +801,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
+	srcPath := path.Join(srcFs.absRoot, srcRemote)
+	dstPath := path.Join(f.absRoot, dstRemote)
 
 	// Check if destination exists
 	ok, err := f.dirExists(dstPath)
@@ -1075,7 +1088,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 
 // path returns the native path of the object
 func (o *Object) path() string {
-	return path.Join(o.fs.root, o.remote)
+	return path.Join(o.fs.absRoot, o.remote)
 }
 
 // setMetadata updates the info in the object from the stat result passed in
@@ -1091,7 +1104,7 @@ func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "stat")
 	}
-	absPath := path.Join(f.root, remote)
+	absPath := path.Join(f.absRoot, remote)
 	info, err = c.sftpClient.Stat(absPath)
 	f.putSftpConnection(&c, err)
 	return info, err

backend/swift/swift.go

@@ -447,6 +447,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		WriteMimeType:     true,
 		BucketBased:       true,
 		BucketBasedRootOK: true,
+		SlowModTime:       true,
 	}).Fill(f)
 	if f.rootContainer != "" && f.rootDirectory != "" {
 		// Check to see if the object exists - ignoring directory markers
@@ -504,7 +505,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, er
 	// making sure we read the full metadata for all 0 byte files.
 	// We don't read the metadata for directory marker objects.
 	if info != nil && info.Bytes == 0 && info.ContentType != "application/directory" {
-		info = nil
+		err := o.readMetaData() // reads info and headers, returning an error
+		if err == fs.ErrorObjectNotFound {
+			// We have a dangling large object here so just return the original metadata
+			fs.Errorf(o, "dangling large object with no contents")
+		} else if err != nil {
+			return nil, err
+		} else {
+			return o, nil
+		}
 	}
 	if info != nil {
 		// Set info but not headers
@@ -536,7 +545,7 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
 // container to the start.
 //
 // Set recurse to read sub directories
-func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, fn listFn) error {
+func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn listFn) error {
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
@@ -570,7 +579,7 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
 				}
-				if remote == prefix {
+				if !includeDirMarkers && remote == prefix {
 					// If we have zero length directory markers ending in / then swift
 					// will return them in the listing for the directory which causes
 					// duplicate directories.  Ignore them here.
@@ -593,8 +602,8 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
 type addEntryFn func(fs.DirEntry) error
 
 // list the objects into the function supplied
-func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, fn addEntryFn) error {
-	err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) {
+func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn addEntryFn) error {
+	err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, includeDirMarkers, func(remote string, object *swift.Object, isDirectory bool) (err error) {
 		if isDirectory {
 			remote = strings.TrimRight(remote, "/")
 			d := fs.NewDir(remote, time.Time{}).SetSize(object.Bytes)
@@ -606,7 +615,7 @@ func (f *Fs) list(container, directory, prefix string, addContainer bool, recurs
 			if err != nil {
 				return err
 			}
-			if o.Storable() {
+			if includeDirMarkers || o.Storable() {
 				err = fn(o)
 			}
 		}
@@ -624,7 +633,7 @@ func (f *Fs) listDir(container, directory, prefix string, addContainer bool) (en
 		return nil, fs.ErrorListBucketRequired
 	}
 	// List the objects
-	err = f.list(container, directory, prefix, addContainer, false, func(entry fs.DirEntry) error {
+	err = f.list(container, directory, prefix, addContainer, false, false, func(entry fs.DirEntry) error {
 		entries = append(entries, entry)
 		return nil
 	})
@@ -694,7 +703,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	container, directory := f.split(dir)
 	list := walk.NewListRHelper(callback)
 	listR := func(container, directory, prefix string, addContainer bool) error {
-		return f.list(container, directory, prefix, addContainer, true, func(entry fs.DirEntry) error {
+		return f.list(container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {
 			return list.Add(entry)
 		})
 	}
@@ -841,7 +850,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 	go func() {
 		delErr <- operations.DeleteFiles(ctx, toBeDeleted)
 	}()
-	err := f.list(f.rootContainer, f.rootDirectory, f.rootDirectory, f.rootContainer == "", true, func(entry fs.DirEntry) error {
+	err := f.list(f.rootContainer, f.rootDirectory, f.rootDirectory, f.rootContainer == "", true, true, func(entry fs.DirEntry) error {
 		if o, ok := entry.(*Object); ok {
 			toBeDeleted <- o
 		}
@@ -1103,7 +1112,7 @@ func min(x, y int64) int64 {
 // if except is passed in then segments with that prefix won't be deleted
 func (o *Object) removeSegments(except string) error {
 	segmentsContainer, prefix, err := o.getSegmentsDlo()
-	err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, func(remote string, object *swift.Object, isDirectory bool) error {
+	err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, true, func(remote string, object *swift.Object, isDirectory bool) error {
 		if isDirectory {
 			return nil
 		}

bin/cross-compile.go

@@ -40,6 +40,8 @@ var (
 )
 
 // GOOS/GOARCH pairs we build for
+//
+// If the GOARCH contains a - it is a synthetic arch with more parameters
 var osarches = []string{
 	"windows/386",
 	"windows/amd64",
@@ -48,15 +50,18 @@ var osarches = []string{
 	"linux/386",
 	"linux/amd64",
 	"linux/arm",
+	"linux/arm-v7",
 	"linux/arm64",
 	"linux/mips",
 	"linux/mipsle",
 	"freebsd/386",
 	"freebsd/amd64",
 	"freebsd/arm",
+	"freebsd/arm-v7",
 	"netbsd/386",
 	"netbsd/amd64",
 	"netbsd/arm",
+	"netbsd/arm-v7",
 	"openbsd/386",
 	"openbsd/amd64",
 	"plan9/386",
@@ -69,6 +74,7 @@ var archFlags = map[string][]string{
 	"386":    {"GO386=387"},
 	"mips":   {"GOMIPS=softfloat"},
 	"mipsle": {"GOMIPS=softfloat"},
+	"arm-v7": {"GOARM=7"},
 }
 
 // runEnv - run a shell command with env
@@ -263,6 +269,15 @@ func cleanupResourceSyso(sysoFilePath string) {
 	}
 }
 
+// Trip a version suffix off the arch if present
+func stripVersion(goarch string) string {
+	i := strings.Index(goarch, "-")
+	if i < 0 {
+		return goarch
+	}
+	return goarch[:i]
+}
+
 // build the binary in dir returning success or failure
 func compileArch(version, goos, goarch, dir string) bool {
 	log.Printf("Compiling %s/%s", goos, goarch)
@@ -290,7 +305,7 @@ func compileArch(version, goos, goarch, dir string) bool {
 	}
 	env := []string{
 		"GOOS=" + goos,
-		"GOARCH=" + goarch,
+		"GOARCH=" + stripVersion(goarch),
 	}
 	if !*cgo {
 		env = append(env, "CGO_ENABLED=0")

bin/get-github-release.go

@@ -374,16 +374,13 @@ func untar(srcFile, fileName, extractDir string) {
 				if err != nil {
 					log.Fatalf("Couldn't open output file: %v", err)
 				}
-				defer func() {
-					err := out.Close()
-					if err != nil {
-						log.Fatalf("Couldn't close output: %v", err)
-					}
-				}()
 				n, err := io.Copy(out, tarReader)
 				if err != nil {
 					log.Fatalf("Couldn't write output file: %v", err)
 				}
+				if err = out.Close(); err != nil {
+					log.Fatalf("Couldn't close output: %v", err)
+				}
 				log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
 			}
 		}

bin/make_manual.py

@@ -6,6 +6,7 @@ conversion into man pages etc.
 
 import os
 import re
+import time
 from datetime import datetime
 
 docpath = "docs/content"
@@ -156,17 +157,19 @@ def read_commands(docpath):
         if command != "rclone.md":
             docs.append(read_command(command))
     return "\n".join(docs)
-    
+
 def main():
     check_docs(docpath)
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
+    build_date = datetime.utcfromtimestamp(
+            int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-""" % datetime.now().strftime("%b %d, %Y"))
+""" % build_date.strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

cmd/check/check.go

@@ -2,24 +2,128 @@ package check
 
 import (
 	"context"
+	"io"
+	"os"
+	"strings"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 // Globals
 var (
-	download = false
-	oneway   = false
+	download     = false
+	oneway       = false
+	combined     = ""
+	missingOnSrc = ""
+	missingOnDst = ""
+	match        = ""
+	differ       = ""
+	errFile      = ""
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by downloading rather than with hash.")
+	AddFlags(cmdFlags)
+}
+
+// AddFlags adds the check flags to the cmdFlags command
+func AddFlags(cmdFlags *pflag.FlagSet) {
 	flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote")
+	flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file")
+	flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file")
+	flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file")
+	flags.StringVarP(cmdFlags, &match, "match", "", match, "Report all matching files to this file")
+	flags.StringVarP(cmdFlags, &differ, "differ", "", differ, "Report all non-matching files to this file")
+	flags.StringVarP(cmdFlags, &errFile, "error", "", errFile, "Report all files with errors (hashing or reading) to this file")
+}
+
+// FlagsHelp describes the flags for the help
+var FlagsHelp = strings.Replace(`
+If you supply the |--one-way| flag, it will only check that files in
+the source match the files in the destination, not the other way
+around. This means that extra files in the destination that are not in
+the source will not be detected.
+
+The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--src-only|
+and |--error| flags write paths, one per line, to the file name (or
+stdout if it is |-|) supplied. What they write is described in the
+help below. For example |--differ| will write all paths which are
+present on both the source and destination but different.
+
+The |--combined| flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- |= path| means path was found in source and destination and was identical
+- |- path| means path was missing on the source, so only in the destination
+- |+ path| means path was missing on the destination, so only in the source
+- |* path| means path was present in source and destination but different.
+- |! path| means there was an error reading or hashing the source or dest.
+`, "|", "`", -1)
+
+// GetCheckOpt gets the options corresponding to the check flags
+func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err error) {
+	closers := []io.Closer{}
+
+	opt = &operations.CheckOpt{
+		Fsrc:   fsrc,
+		Fdst:   fdst,
+		OneWay: oneway,
+	}
+
+	open := func(name string, pout *io.Writer) error {
+		if name == "" {
+			return nil
+		}
+		if name == "-" {
+			*pout = os.Stdout
+			return nil
+		}
+		out, err := os.Create(name)
+		if err != nil {
+			return err
+		}
+		*pout = out
+		closers = append(closers, out)
+		return nil
+	}
+
+	if err = open(combined, &opt.Combined); err != nil {
+		return nil, nil, err
+	}
+	if err = open(missingOnSrc, &opt.MissingOnSrc); err != nil {
+		return nil, nil, err
+	}
+	if err = open(missingOnDst, &opt.MissingOnDst); err != nil {
+		return nil, nil, err
+	}
+	if err = open(match, &opt.Match); err != nil {
+		return nil, nil, err
+	}
+	if err = open(differ, &opt.Differ); err != nil {
+		return nil, nil, err
+	}
+	if err = open(errFile, &opt.Error); err != nil {
+		return nil, nil, err
+	}
+
+	close = func() {
+		for _, closer := range closers {
+			err := closer.Close()
+			if err != nil {
+				fs.Errorf(nil, "Failed to close report output: %v", err)
+			}
+		}
+	}
+
+	return opt, close, nil
+
 }
 
 var commandDefinition = &cobra.Command{
@@ -37,19 +141,20 @@ If you supply the --download flag, it will download the data from
 both remotes and check them against each other on the fly.  This can
 be useful for remotes that don't support hashes or if you really want
 to check all the data.
-
-If you supply the --one-way flag, it will only check that files in source
-match the files in destination, not the other way around. Meaning extra files in
-destination that are not in the source will not trigger an error.
-`,
+` + FlagsHelp,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, fdst := cmd.NewFsSrcDst(args)
 		cmd.Run(false, true, command, func() error {
-			if download {
-				return operations.CheckDownload(context.Background(), fdst, fsrc, oneway)
+			opt, close, err := GetCheckOpt(fsrc, fdst)
+			if err != nil {
+				return err
 			}
-			return operations.Check(context.Background(), fdst, fsrc, oneway)
+			defer close()
+			if download {
+				return operations.CheckDownload(context.Background(), opt)
+			}
+			return operations.Check(context.Background(), opt)
 		})
 	},
 }

cmd/cmount/mount_test.go

@@ -11,11 +11,9 @@ package cmount
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/vfs/vfstest"
 )
 
 func TestMount(t *testing.T) {
-	testy.SkipUnreliable(t)
 	vfstest.RunTests(t, false, mount)
 }

cmd/cryptcheck/cryptcheck.go

@@ -6,22 +6,17 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/crypt"
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/check"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
 
-// Globals
-var (
-	oneway = false
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlag := commandDefinition.Flags()
-	flags.BoolVarP(cmdFlag, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on destination")
+	check.AddFlags(cmdFlag)
 }
 
 var commandDefinition = &cobra.Command{
@@ -50,11 +45,7 @@ the files in remote:path.
     rclone cryptcheck remote:path encryptedremote:path
 
 After it has run it will log the status of the encryptedremote:.
-
-If you supply the --one-way flag, it will only check that files in source
-match the files in destination, not the other way around. Meaning extra files in
-destination that are not in the source will not trigger an error.
-`,
+` + check.FlagsHelp,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, fdst := cmd.NewFsSrcDst(args)
@@ -79,39 +70,40 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
 	}
 	fs.Infof(nil, "Using %v for hash comparisons", hashType)
 
+	opt, close, err := check.GetCheckOpt(fsrc, fcrypt)
+	if err != nil {
+		return err
+	}
+	defer close()
+
 	// checkIdentical checks to see if dst and src are identical
 	//
 	// it returns true if differences were found
 	// it also returns whether it couldn't be hashed
-	checkIdentical := func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
+	opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
 		cryptDst := dst.(*crypt.Object)
 		underlyingDst := cryptDst.UnWrap()
 		underlyingHash, err := underlyingDst.Hash(ctx, hashType)
 		if err != nil {
-			err = fs.CountError(err)
-			fs.Errorf(dst, "Error reading hash from underlying %v: %v", underlyingDst, err)
-			return true, false
+			return true, false, errors.Wrapf(err, "error reading hash from underlying %v", underlyingDst)
 		}
 		if underlyingHash == "" {
-			return false, true
+			return false, true, nil
 		}
 		cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)
 		if err != nil {
-			err = fs.CountError(err)
-			fs.Errorf(dst, "Error computing hash: %v", err)
-			return true, false
+			return true, false, errors.Wrap(err, "error computing hash")
 		}
 		if cryptHash == "" {
-			return false, true
+			return false, true, nil
 		}
 		if cryptHash != underlyingHash {
 			err = errors.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)
-			err = fs.CountError(err)
 			fs.Errorf(src, err.Error())
-			return true, false
+			return true, false, nil
 		}
-		return false, false
+		return false, false, nil
 	}
 
-	return operations.CheckFn(ctx, fcrypt, fsrc, checkIdentical, oneway)
+	return operations.CheckFn(ctx, opt)
 }

cmd/mkdir/mkdir.go

@@ -2,8 +2,10 @@ package mkdir
 
 import (
 	"context"
+	"strings"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
@@ -18,6 +20,9 @@ var commandDefinition = &cobra.Command{
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
+		if !fdst.Features().CanHaveEmptyDirectories && strings.Contains(fdst.Root(), "/") {
+			fs.Logf(fdst, "Warning: running mkdir on a remote which can't have empty directories does nothing")
+		}
 		cmd.Run(true, false, command, func() error {
 			return operations.Mkdir(context.Background(), fdst, "")
 		})

cmd/mount2/mount_test.go

@@ -5,11 +5,9 @@ package mount2
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/vfs/vfstest"
 )
 
 func TestMount(t *testing.T) {
-	testy.SkipUnreliable(t)
 	vfstest.RunTests(t, false, mount)
 }

cmd/serve/ftp/ftp.go

@@ -1,17 +1,15 @@
 // Package ftp implements an FTP server for rclone
 
-//+build !plan9
+//+build !plan9,go1.13
 
 package ftp
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"net"
 	"os"
 	"os/user"
-	"runtime"
 	"strconv"
 	"sync"
 
@@ -28,7 +26,7 @@ import (
 	"github.com/rclone/rclone/vfs/vfsflags"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-	ftp "goftp.io/server"
+	ftp "goftp.io/server/core"
 )
 
 // Options contains options for the http Server
@@ -114,13 +112,11 @@ You can set a single username and password with the --user and --pass flags.
 
 // server contains everything to run the server
 type server struct {
-	f         fs.Fs
-	srv       *ftp.Server
-	opt       Options
-	vfs       *vfs.VFS
-	proxy     *proxy.Proxy
-	pendingMu sync.Mutex
-	pending   map[string]*Driver // pending Driver~s that haven't got their VFS
+	f     fs.Fs
+	srv   *ftp.Server
+	opt   Options
+	vfs   *vfs.VFS
+	proxy *proxy.Proxy
 }
 
 // Make a new FTP to serve the remote
@@ -135,9 +131,8 @@ func newServer(f fs.Fs, opt *Options) (*server, error) {
 	}
 
 	s := &server{
-		f:       f,
-		opt:     *opt,
-		pending: make(map[string]*Driver),
+		f:   f,
+		opt: *opt,
 	}
 	if proxyflags.Opt.AuthProxy != "" {
 		s.proxy = proxy.New(&proxyflags.Opt)
@@ -151,7 +146,7 @@ func newServer(f fs.Fs, opt *Options) (*server, error) {
 		Factory:        s, // implemented by NewDriver method
 		Hostname:       host,
 		Port:           portNum,
-		PublicIp:       opt.PublicIP,
+		PublicIP:       opt.PublicIP,
 		PassivePorts:   opt.PassivePorts,
 		Auth:           s, // implemented by CheckPasswd method
 		Logger:         &Logger{},
@@ -200,78 +195,13 @@ func (l *Logger) PrintResponse(sessionID string, code int, message string) {
 	fs.Infof(sessionID, "< %d %s", code, message)
 }
 
-// findID finds the connection ID of the calling program.  It does
-// this in an incredibly hacky way by looking in the stack trace.
-//
-// callerName should be the name of the function that we are looking
-// for with a trailing '('
-//
-// What is really needed is a change of calling protocol so
-// CheckPassword is called with the connection.
-func findID(callerName []byte) (string, error) {
-	// Dump the stack in this format
-	// github.com/rclone/rclone/vendor/goftp.io/server.(*Conn).Serve(0xc0000b2680)
-	// 	/home/ncw/go/src/github.com/rclone/rclone/vendor/goftp.io/server/conn.go:116 +0x11d
-	buf := make([]byte, 4096)
-	n := runtime.Stack(buf, false)
-	buf = buf[:n]
-
-	// look for callerName first
-	i := bytes.Index(buf, callerName)
-	if i < 0 {
-		return "", errors.Errorf("findID: caller name not found in:\n%s", buf)
-	}
-	buf = buf[i+len(callerName):]
-
-	// find next ')'
-	i = bytes.IndexByte(buf, ')')
-	if i < 0 {
-		return "", errors.Errorf("findID: end of args not found in:\n%s", buf)
-	}
-	buf = buf[:i]
-
-	// trim off first argument
-	// find next ','
-	i = bytes.IndexByte(buf, ',')
-	if i >= 0 {
-		buf = buf[:i]
-	}
-
-	return string(buf), nil
-}
-
-var connServeFunction = []byte("(*Conn).Serve(")
-
 // CheckPasswd handle auth based on configuration
+//
+// This is not used - the one in Driver should be called instead
 func (s *server) CheckPasswd(user, pass string) (ok bool, err error) {
-	var VFS *vfs.VFS
-	if s.proxy != nil {
-		VFS, _, err = s.proxy.Call(user, pass, false)
-		if err != nil {
-			fs.Infof(nil, "proxy login failed: %v", err)
-			return false, nil
-		}
-		id, err := findID(connServeFunction)
-		if err != nil {
-			fs.Infof(nil, "proxy login failed: failed to read ID from stack: %v", err)
-			return false, nil
-		}
-		s.pendingMu.Lock()
-		d := s.pending[id]
-		delete(s.pending, id)
-		s.pendingMu.Unlock()
-		if d == nil {
-			return false, errors.Errorf("proxy login failed: failed to find pending Driver under ID %q", id)
-		}
-		d.vfs = VFS
-	} else {
-		ok = s.opt.BasicUser == user && (s.opt.BasicPass == "" || s.opt.BasicPass == pass)
-		if !ok {
-			fs.Infof(nil, "login failed: bad credentials")
-			return false, nil
-		}
-	}
-	return true, nil
+	err = errors.New("internal error: server.CheckPasswd should never be called")
+	fs.Errorf(nil, "Error: %v", err)
+	return false, err
 }
 
 // NewDriver starts a new session for each client connection
@@ -291,15 +221,25 @@ type Driver struct {
 	lock sync.Mutex
 }
 
-//Init a connection
-func (d *Driver) Init(c *ftp.Conn) {
-	defer log.Trace("", "Init session")("")
-	if d.s.proxy != nil {
-		id := fmt.Sprintf("%p", c)
-		d.s.pendingMu.Lock()
-		d.s.pending[id] = d
-		d.s.pendingMu.Unlock()
+// CheckPasswd handle auth based on configuration
+func (d *Driver) CheckPasswd(user, pass string) (ok bool, err error) {
+	s := d.s
+	if s.proxy != nil {
+		var VFS *vfs.VFS
+		VFS, _, err = s.proxy.Call(user, pass, false)
+		if err != nil {
+			fs.Infof(nil, "proxy login failed: %v", err)
+			return false, nil
+		}
+		d.vfs = VFS
+	} else {
+		ok = s.opt.BasicUser == user && (s.opt.BasicPass == "" || s.opt.BasicPass == pass)
+		if !ok {
+			fs.Infof(nil, "login failed: bad credentials")
+			return false, nil
+		}
 	}
+	return true, nil
 }
 
 //Stat get information on file or folder

cmd/serve/ftp/ftp_test.go

@@ -3,12 +3,11 @@
 //
 // We skip tests on platforms with troublesome character mappings
 
-//+build !windows,!darwin,!plan9
+//+build !windows,!darwin,!plan9,go1.13
 
 package ftp
 
 import (
-	"fmt"
 	"testing"
 
 	_ "github.com/rclone/rclone/backend/local"
@@ -17,8 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	ftp "goftp.io/server"
+	ftp "goftp.io/server/core"
 )
 
 const (
@@ -70,10 +68,3 @@ func TestFTP(t *testing.T) {
 
 	servetest.Run(t, "ftp", start)
 }
-
-func TestFindID(t *testing.T) {
-	id, err := findID([]byte("TestFindID("))
-	require.NoError(t, err)
-	// id should be the argument to this function
-	assert.Equal(t, fmt.Sprintf("%p", t), id)
-}

cmd/serve/ftp/ftp_unsupported.go

@@ -1,7 +1,7 @@
 // Build  for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9
+// +build plan9 !go1.13
 
 package ftp
 

docs/content/authors.md

@@ -389,3 +389,9 @@ put them back in again.` >}}
   * jtagcat <gitlab@c7.ee>
   * Petri Salminen <petri@salminen.dev>
   * Tim Burke <tim.burke@gmail.com>
+  * Kai Lüke <kai@kinvolk.io>
+  * Garrett Squire <github@garrettsquire.com>
+  * Evan Harris <eharris@puremagic.com>
+  * Kevin <keyam@microsoft.com>
+  * Morten Linderud <morten@linderud.pw>
+  * Dmitry Ustalov <dmitry.ustalov@gmail.com>

docs/content/b2.md

@@ -117,6 +117,12 @@ the following characters are also replaced:
 Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
 as they can't be used in JSON strings.
 
+Note that in 2020-05 Backblaze started allowing \ characters in file
+names. Rclone hasn't changed its encoding as this could cause syncs to
+re-transfer files. If you want rclone not to replace \ then see the
+`--b2-encoding` flag below and remove the `BackSlash` from the
+string. This can be set in the config.
+
 ### SHA1 checksums ###
 
 The SHA1 checksums of the files are checked on upload and download and

docs/content/docs.md

@@ -1849,6 +1849,27 @@ mys3:
 Note that if you want to create a remote using environment variables
 you must create the `..._TYPE` variable as above.
 
+### Precedence
+
+The various different methods of backend configuration are read in
+this order and the first one with a value is used.
+
+- Flag values as supplied on the command line, eg `--drive-use-trash`.
+- Remote specific environment vars, eg `RCLONE_CONFIG_MYREMOTE_USE_TRASH` (see above).
+- Backend specific environment vars, eg `RCLONE_DRIVE_USE_TRASH`.
+- Config file, eg `use_trash = false`.
+- Default values, eg `true` - these can't be changed.
+
+So if both `--drive-use-trash` is supplied on the config line and an
+environment variable `RCLONE_DRIVE_USE_TRASH` is set, the command line
+flag will take preference.
+
+For non backend configuration the order is as follows:
+
+- Flag values as supplied on the command line, eg `--stats 5s`.
+- Environment vars, eg `RCLONE_STATS=5s`.
+- Default values, eg `1m` - these can't be changed.
+
 ### Other environment variables ###
 
 - `RCLONE_CONFIG_PASS` set to contain your config file password (see [Configuration Encryption](#configuration-encryption) section)

docs/content/downloads.md

@@ -11,7 +11,8 @@ Rclone Download {{< version >}}
 |:-------:|:-------:|:-----:|:-----:|:----:|:----:|:-------:|:------:|:-------:|:-----:|:-------:|
 | Intel/AMD - 64 Bit | {{< download windows amd64 >}} | {{< download osx amd64 >}} | {{< download linux amd64 >}} | {{< download linux amd64 deb >}} | {{< download linux amd64 rpm >}} | {{< download freebsd amd64 >}} | {{< download netbsd amd64 >}} | {{< download openbsd amd64 >}} | {{< download plan9 amd64 >}} | {{< download solaris amd64 >}} |
 | Intel/AMD - 32 Bit | {{< download windows 386 >}} | {{< download osx 386 >}} | {{< download linux 386 >}} | {{< download linux 386 deb >}} | {{< download linux 386 rpm >}} | {{< download freebsd 386 >}} | {{< download netbsd 386 >}} | {{< download openbsd 386 >}} | {{< download plan9 386 >}} | - |
-| ARM - 32 Bit | - | - | {{< download linux arm >}} | {{< download linux arm deb >}} | {{< download linux arm rpm >}} | {{< download freebsd arm >}} | {{< download netbsd arm >}} | - | - | - |
+| ARMv6 - 32 Bit | - | - | {{< download linux arm >}} | {{< download linux arm deb >}} | {{< download linux arm rpm >}} | {{< download freebsd arm >}} | {{< download netbsd arm >}} | - | - | - |
+| ARMv7 - 32 Bit | - | - | {{< download linux arm-v7 >}} | {{< download linux arm-v7 deb >}} | {{< download linux arm-v7 rpm >}} | {{< download freebsd arm-v7 >}} | {{< download netbsd arm-v7 >}} | - | - | - |
 | ARM - 64 Bit | - | - | {{< download linux arm64 >}} | {{< download linux arm64 deb >}} | {{< download linux arm64 rpm >}} | - | - | - | - | - |
 | MIPS - Big Endian | - | - | {{< download linux mips >}} | {{< download linux mips deb >}} | {{< download linux mips rpm >}} | - | - | - | - | - |
 | MIPS - Little Endian | - | - | {{< download linux mipsle >}} | {{< download linux mipsle deb >}} | {{< download linux mipsle rpm >}} | - | - | - | - | - |
@@ -70,7 +71,8 @@ script) from a URL which doesn't change then you can use these links.
 |:-------:|:-------:|:-----:|:-----:|:----:|:----:|:-------:|:------:|:-------:|:-----:|:-------:|
 | Intel/AMD - 64 Bit | {{< cdownload windows amd64 >}} | {{< cdownload osx amd64 >}} | {{< cdownload linux amd64 >}} | {{< cdownload linux amd64 deb >}} | {{< cdownload linux amd64 rpm >}} | {{< cdownload freebsd amd64 >}} | {{< cdownload netbsd amd64 >}} | {{< cdownload openbsd amd64 >}} | {{< cdownload plan9 amd64 >}} | {{< cdownload solaris amd64 >}} |
 | Intel/AMD - 32 Bit | {{< cdownload windows 386 >}} | {{< cdownload osx 386 >}} | {{< cdownload linux 386 >}} | {{< cdownload linux 386 deb >}} | {{< cdownload linux 386 rpm >}} | {{< cdownload freebsd 386 >}} | {{< cdownload netbsd 386 >}} | {{< cdownload openbsd 386 >}} | {{< cdownload plan9 386 >}} | - |
-| ARM - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
+| ARMv6 - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
+| ARMv7 - 32 Bit | - | - | {{< cdownload linux arm-v7 >}} | {{< cdownload linux arm-v7 deb >}} | {{< cdownload linux arm-v7 rpm >}} | {{< cdownload freebsd arm-v7 >}} | {{< cdownload netbsd arm-v7 >}} | - | - | - |
 | ARM - 64 Bit | - | - | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
 | MIPS - Big Endian | - | - | {{< cdownload linux mips >}} | {{< cdownload linux mips deb >}} | {{< cdownload linux mips rpm >}} | - | - | - | - | - |
 | MIPS - Little Endian | - | - | {{< cdownload linux mipsle >}} | {{< cdownload linux mipsle deb >}} | {{< cdownload linux mipsle rpm >}} | - | - | - | - | - |

docs/content/drive.md

@@ -1197,3 +1197,10 @@ for rclone to be able to get its token-id (but as this only happens during
 the remote configuration, it's not such a big deal). 
 
 (Thanks to @balazer on github for these instructions.)
+
+Sometimes, creation of an OAuth consent in Google API Console fails due to an error message
+“The request failed because changes to one of the field of the resource is not supported”.
+As a convenient workaround, the necessary Google Drive API key can be created on the
+[Python Quickstart](https://developers.google.com/drive/api/v3/quickstart/python) page.
+Just push the Enable the Drive API button to receive the Client ID and Secret.
+Note that it will automatically create a new project in the API Console.

docs/content/ftp.md

@@ -10,6 +10,10 @@ FTP is the File Transfer Protocol. FTP support is provided using the
 [github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
 package.
 
+Paths are specified as `remote:path`. If the path does not begin with
+a `/` it is relative to the home directory of the user.  An empty path
+`remote:` refers to the user's home directory.
+
 Here is an example of making an FTP configuration.  First run
 
     rclone config
@@ -253,8 +257,9 @@ See: the [encoding section in the overview](/overview/#encoding) for more info.
 
 ### Limitations ###
 
-Note that since FTP isn't HTTP based the following flags don't work
-with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
+Note that FTP does have its own implementation of : `--dump headers`,
+`--dump bodies`, `--dump auth` for debugging which isn't the same as
+the HTTP based backends - it has less fine grained control.
 
 Note that `--timeout` isn't supported (but `--contimeout` is).
 

docs/content/googlecloudstorage.md

@@ -194,6 +194,13 @@ the rclone config file, you can set `service_account_credentials` with
 the actual contents of the file instead, or set the equivalent
 environment variable.
 
+### Anonymous Access ###
+
+For downloads of objects that permit public access you can configure rclone
+to use anonymous access by setting `anonymous` to `true`.
+With unauthorized access you can't write or create files but only read or list
+those buckets and objects that have public read access.
+
 ### Application Default Credentials ###
 
 If no other source of credentials is provided, rclone will fall back

docs/content/install.md

@@ -174,7 +174,7 @@ kill %1
 
 ## Install from source ##
 
-Make sure you have at least [Go](https://golang.org/) 1.7
+Make sure you have at least [Go](https://golang.org/) 1.11
 installed.  [Download go](https://golang.org/dl/) if necessary.  The
 latest release is recommended. Then
 
@@ -183,16 +183,14 @@ latest release is recommended. Then
     go build
     ./rclone version
 
-You can also build and install rclone in the
-[GOPATH](https://github.com/golang/go/wiki/GOPATH) (which defaults to
-`~/go`) with:
+This will leave you a checked out version of rclone you can modify.
+
+You can also build rclone with:
 
     go get -u -v github.com/rclone/rclone
 
 and this will build the binary in `$GOPATH/bin` (`~/go/bin/rclone` by
-default) after downloading the source to
-`$GOPATH/src/github.com/rclone/rclone` (`~/go/src/github.com/rclone/rclone`
-by default).
+default) after downloading the source to the go module cache..
 
 ## Installation with Ansible ##
 

docs/content/onedrive.md

@@ -127,7 +127,7 @@ requests.
 If you are having problems with them (E.g., seeing a lot of throttling), you can get your own
 Client ID and Key by following the steps below:
 
-1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade, then click `New registration`.
+1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI` Enter `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Copy and keep that secret for later use.
 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.

docs/content/privacy.md

@@ -53,6 +53,14 @@ Users are advised to use social media platforms wisely and communicate / engage
 
 This website may use social sharing buttons which help share web content directly from web pages to the social media platform in question. Users are advised before using such social sharing buttons that they do so at their own discretion and note that the social media platform may track and save your request to share a web page respectively through your social media platform account.
 
+## Use of Cloud API User Data ##
+
+Rclone is a command line program to manage files on cloud storage. Its sole purpose is to access and manipulate user content in the [supported](/overview/) cloud storage systems from a local machine of the end user. For accessing the user content via the cloud provider API, Rclone uses authentication mechanisms, such as OAuth or HTTP Cookies, depending on the particular cloud provider offerings. Use of these authentication mechanisms and user data is governed by the privacy policies mentioned in the [Resources & Further Information](/privacy/#resources-further-information) section and followed by the privacy policy of Rclone.
+
+* Rclone provides the end user with access to their files available in a storage system associated by the authentication credentials via the publicly exposed API of the storage system.
+* Rclone allows storing the authentication credentials on the user machine in the local configuration file.
+* Rclone does not share any user data with third parties.
+
 ## Resources & Further Information ##
 
   * [Data Protection Act 1998](http://www.legislation.gov.uk/ukpga/1998/29/contents)
@@ -61,4 +69,5 @@ This website may use social sharing buttons which help share web content directl
   * [Twitter Privacy Policy](https://twitter.com/privacy)
   * [Facebook Privacy Policy](https://www.facebook.com/about/privacy/)
   * [Google Privacy Policy](https://www.google.com/privacy.html)
+  * [Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy)
   * [Sample Website Privacy Policy](http://www.jamieking.co.uk/resources/free_sample_privacy_policy.html)

fs/accounting/accounting.go

@@ -109,6 +109,14 @@ func (acc *Account) WithBuffer() *Account {
 	return acc
 }
 
+// HasBuffer - returns true if this Account has an AsyncReader with a buffer
+func (acc *Account) HasBuffer() bool {
+	acc.mu.Lock()
+	defer acc.mu.Unlock()
+	_, ok := acc.in.(*asyncreader.AsyncReader)
+	return ok
+}
+
 // GetReader returns the underlying io.ReadCloser under any Buffer
 func (acc *Account) GetReader() io.ReadCloser {
 	acc.mu.Lock()

fs/accounting/accounting_test.go

@@ -37,6 +37,7 @@ func TestNewAccountSizeName(t *testing.T) {
 	assert.Equal(t, acc, stats.inProgress.get("test"))
 	acc.Done()
 	assert.Nil(t, stats.inProgress.get("test"))
+	assert.False(t, acc.HasBuffer())
 }
 
 func TestAccountWithBuffer(t *testing.T) {
@@ -44,7 +45,9 @@ func TestAccountWithBuffer(t *testing.T) {
 
 	stats := NewStats()
 	acc := newAccountSizeName(stats, in, -1, "test")
+	assert.False(t, acc.HasBuffer())
 	acc.WithBuffer()
+	assert.True(t, acc.HasBuffer())
 	// should have a buffer for an unknown size
 	_, ok := acc.in.(*asyncreader.AsyncReader)
 	require.True(t, ok)

fs/accounting/stats.go

@@ -26,11 +26,11 @@ type StatsInfo struct {
 	retryError        bool
 	retryAfter        time.Time
 	checks            int64
-	checking          *stringSet
+	checking          *transferMap
 	checkQueue        int
 	checkQueueSize    int64
 	transfers         int64
-	transferring      *stringSet
+	transferring      *transferMap
 	transferQueue     int
 	transferQueueSize int64
 	renames           int64
@@ -47,8 +47,8 @@ type StatsInfo struct {
 // NewStats creates an initialised StatsInfo
 func NewStats() *StatsInfo {
 	return &StatsInfo{
-		checking:     newStringSet(fs.Config.Checkers, "checking"),
-		transferring: newStringSet(fs.Config.Transfers, "transferring"),
+		checking:     newTransferMap(fs.Config.Checkers, "checking"),
+		transferring: newTransferMap(fs.Config.Transfers, "transferring"),
 		inProgress:   newInProgress(),
 	}
 }
@@ -72,8 +72,8 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 		var c []string
 		s.checking.mu.RLock()
 		defer s.checking.mu.RUnlock()
-		for name := range s.checking.items {
-			c = append(c, name)
+		for _, tr := range s.checking.sortedSlice() {
+			c = append(c, tr.remote)
 		}
 		out["checking"] = c
 	}
@@ -81,11 +81,11 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 		s.transferring.mu.RLock()
 
 		var t []rc.Params
-		for name := range s.transferring.items {
-			if acc := s.inProgress.get(name); acc != nil {
+		for _, tr := range s.transferring.sortedSlice() {
+			if acc := s.inProgress.get(tr.remote); acc != nil {
 				t = append(t, acc.RemoteStats())
 			} else {
-				t = append(t, s.transferRemoteStats(name))
+				t = append(t, s.transferRemoteStats(tr))
 			}
 		}
 		out["transferring"] = t
@@ -108,18 +108,11 @@ func (s *StatsInfo) Speed() float64 {
 	return speed
 }
 
-func (s *StatsInfo) transferRemoteStats(name string) rc.Params {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	for _, tr := range s.startedTransfers {
-		if tr.remote == name {
-			return rc.Params{
-				"name": name,
-				"size": tr.size,
-			}
-		}
+func (s *StatsInfo) transferRemoteStats(tr *Transfer) rc.Params {
+	return rc.Params{
+		"name": tr.remote,
+		"size": tr.size,
 	}
-	return rc.Params{"name": name}
 }
 
 // timeRange is a start and end time of a transfer
@@ -558,8 +551,9 @@ func (s *StatsInfo) RetryAfter() time.Time {
 
 // NewCheckingTransfer adds a checking transfer to the stats, from the object.
 func (s *StatsInfo) NewCheckingTransfer(obj fs.Object) *Transfer {
-	s.checking.add(obj.Remote())
-	return newCheckingTransfer(s, obj)
+	tr := newCheckingTransfer(s, obj)
+	s.checking.add(tr)
+	return tr
 }
 
 // DoneChecking removes a check from the stats
@@ -579,14 +573,16 @@ func (s *StatsInfo) GetTransfers() int64 {
 
 // NewTransfer adds a transfer to the stats from the object.
 func (s *StatsInfo) NewTransfer(obj fs.Object) *Transfer {
-	s.transferring.add(obj.Remote())
-	return newTransfer(s, obj)
+	tr := newTransfer(s, obj)
+	s.transferring.add(tr)
+	return tr
 }
 
 // NewTransferRemoteSize adds a transfer to the stats based on remote and size.
 func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer {
-	s.transferring.add(remote)
-	return newTransferRemoteSize(s, remote, size, false)
+	tr := newTransferRemoteSize(s, remote, size, false)
+	s.transferring.add(tr)
+	return tr
 }
 
 // DoneTransferring removes a transfer from the stats

fs/accounting/stats_groups.go

@@ -104,8 +104,8 @@ Returns the following values:
 				"eta": estimated time in seconds until file transfer completion
 				"name": name of the file,
 				"percentage": progress of the file transfer in percent,
-				"speed": speed in bytes/sec,
-				"speedAvg": speed in bytes/sec as an exponentially weighted moving average,
+				"speed": average speed over the whole transfer in bytes/sec,
+				"speedAvg": current speed in bytes/sec as an exponentially weighted moving average,
 				"size": size of the file in bytes
 			}
 		],

fs/accounting/stringset.go

@@ -1,112 +0,0 @@
-package accounting
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/rclone/rclone/fs"
-)
-
-// stringSet holds a set of strings
-type stringSet struct {
-	mu    sync.RWMutex
-	items map[string]struct{}
-	name  string
-}
-
-// newStringSet creates a new empty string set of capacity size
-func newStringSet(size int, name string) *stringSet {
-	return &stringSet{
-		items: make(map[string]struct{}, size),
-		name:  name,
-	}
-}
-
-// add adds remote to the set
-func (ss *stringSet) add(remote string) {
-	ss.mu.Lock()
-	ss.items[remote] = struct{}{}
-	ss.mu.Unlock()
-}
-
-// del removes remote from the set
-func (ss *stringSet) del(remote string) {
-	ss.mu.Lock()
-	delete(ss.items, remote)
-	ss.mu.Unlock()
-}
-
-// merge adds items from another set
-func (ss *stringSet) merge(m *stringSet) {
-	ss.mu.Lock()
-	m.mu.Lock()
-	for item := range m.items {
-		ss.items[item] = struct{}{}
-	}
-	m.mu.Unlock()
-	ss.mu.Unlock()
-}
-
-// empty returns whether the set has any items
-func (ss *stringSet) empty() bool {
-	ss.mu.RLock()
-	defer ss.mu.RUnlock()
-	return len(ss.items) == 0
-}
-
-// count returns the number of items in the set
-func (ss *stringSet) count() int {
-	ss.mu.RLock()
-	defer ss.mu.RUnlock()
-	return len(ss.items)
-}
-
-// String returns string representation of set items excluding any in
-// exclude (if set).
-func (ss *stringSet) String(progress *inProgress, exclude *stringSet) string {
-	ss.mu.RLock()
-	defer ss.mu.RUnlock()
-	strngs := make([]string, 0, len(ss.items))
-	for name := range ss.items {
-		if exclude != nil {
-			exclude.mu.RLock()
-			_, found := exclude.items[name]
-			exclude.mu.RUnlock()
-			if found {
-				continue
-			}
-		}
-		var out string
-		if acc := progress.get(name); acc != nil {
-			out = acc.String()
-		} else {
-			out = fmt.Sprintf("%*s: %s",
-				fs.Config.StatsFileNameLength,
-				shortenName(name, fs.Config.StatsFileNameLength),
-				ss.name,
-			)
-		}
-		strngs = append(strngs, " * "+out)
-	}
-	sorted := sort.StringSlice(strngs)
-	sorted.Sort()
-	return strings.Join(sorted, "\n")
-}
-
-// progress returns total bytes read as well as the size.
-func (ss *stringSet) progress(stats *StatsInfo) (totalBytes, totalSize int64) {
-	ss.mu.RLock()
-	defer ss.mu.RUnlock()
-	for name := range ss.items {
-		if acc := stats.inProgress.get(name); acc != nil {
-			bytes, size := acc.progress()
-			if size >= 0 && bytes >= 0 {
-				totalBytes += bytes
-				totalSize += size
-			}
-		}
-	}
-	return totalBytes, totalSize
-}

fs/accounting/transfermap.go

@@ -0,0 +1,124 @@
+package accounting
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// transferMap holds name to transfer map
+type transferMap struct {
+	mu    sync.RWMutex
+	items map[string]*Transfer
+	name  string
+}
+
+// newTransferMap creates a new empty transfer map of capacity size
+func newTransferMap(size int, name string) *transferMap {
+	return &transferMap{
+		items: make(map[string]*Transfer, size),
+		name:  name,
+	}
+}
+
+// add adds a new transfer to the map
+func (tm *transferMap) add(tr *Transfer) {
+	tm.mu.Lock()
+	tm.items[tr.remote] = tr
+	tm.mu.Unlock()
+}
+
+// del removes a transfer from the map by name
+func (tm *transferMap) del(remote string) {
+	tm.mu.Lock()
+	delete(tm.items, remote)
+	tm.mu.Unlock()
+}
+
+// merge adds items from another map
+func (tm *transferMap) merge(m *transferMap) {
+	tm.mu.Lock()
+	m.mu.Lock()
+	for name, tr := range m.items {
+		tm.items[name] = tr
+	}
+	m.mu.Unlock()
+	tm.mu.Unlock()
+}
+
+// empty returns whether the map has any items
+func (tm *transferMap) empty() bool {
+	tm.mu.RLock()
+	defer tm.mu.RUnlock()
+	return len(tm.items) == 0
+}
+
+// count returns the number of items in the map
+func (tm *transferMap) count() int {
+	tm.mu.RLock()
+	defer tm.mu.RUnlock()
+	return len(tm.items)
+}
+
+// sortedSlice returns all transfers sorted by start time
+func (tm *transferMap) sortedSlice() []*Transfer {
+	tm.mu.RLock()
+	defer tm.mu.RUnlock()
+	s := make([]*Transfer, 0, len(tm.items))
+	for _, tr := range tm.items {
+		s = append(s, tr)
+	}
+	sort.Slice(s, func(i, j int) bool {
+		return s[i].startedAt.Before(s[j].startedAt)
+	})
+	return s
+}
+
+// String returns string representation of map items excluding any in
+// exclude (if set).
+func (tm *transferMap) String(progress *inProgress, exclude *transferMap) string {
+	tm.mu.RLock()
+	defer tm.mu.RUnlock()
+	strngs := make([]string, 0, len(tm.items))
+	for _, tr := range tm.sortedSlice() {
+		if exclude != nil {
+			exclude.mu.RLock()
+			_, found := exclude.items[tr.remote]
+			exclude.mu.RUnlock()
+			if found {
+				continue
+			}
+		}
+		var out string
+		if acc := progress.get(tr.remote); acc != nil {
+			out = acc.String()
+		} else {
+			out = fmt.Sprintf("%*s: %s",
+				fs.Config.StatsFileNameLength,
+				shortenName(tr.remote, fs.Config.StatsFileNameLength),
+				tm.name,
+			)
+		}
+		strngs = append(strngs, " * "+out)
+	}
+	return strings.Join(strngs, "\n")
+}
+
+// progress returns total bytes read as well as the size.
+func (tm *transferMap) progress(stats *StatsInfo) (totalBytes, totalSize int64) {
+	tm.mu.RLock()
+	defer tm.mu.RUnlock()
+	for name := range tm.items {
+		if acc := stats.inProgress.get(name); acc != nil {
+			bytes, size := acc.progress()
+			if size >= 0 && bytes >= 0 {
+				totalBytes += bytes
+				totalSize += size
+			}
+		}
+	}
+	return totalBytes, totalSize
+}

fs/cache/cache.go

@@ -14,9 +14,9 @@ var (
 	remap = map[string]string{} // map user supplied names to canonical names
 )
 
-// Lookup fsString in the mapping from user supplied names to
-// canonical names and return the canonical form
-func canonicalize(fsString string) string {
+// Canonicalize looks up fsString in the mapping from user supplied
+// names to canonical names and return the canonical form
+func Canonicalize(fsString string) string {
 	mu.Lock()
 	canonicalName, ok := remap[fsString]
 	mu.Unlock()
@@ -40,7 +40,7 @@ func addMapping(fsString, canonicalName string) {
 // GetFn gets an fs.Fs named fsString either from the cache or creates
 // it afresh with the create function
 func GetFn(fsString string, create func(fsString string) (fs.Fs, error)) (f fs.Fs, err error) {
-	fsString = canonicalize(fsString)
+	fsString = Canonicalize(fsString)
 	created := false
 	value, err := c.Get(fsString, func(fsString string) (f interface{}, ok bool, err error) {
 		f, err = create(fsString)
@@ -56,12 +56,20 @@ func GetFn(fsString string, create func(fsString string) (fs.Fs, error)) (f fs.F
 	if created {
 		canonicalName := fs.ConfigString(f)
 		if canonicalName != fsString {
-			fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", fsString, canonicalName)
-			value, found := c.Rename(fsString, canonicalName)
-			if found {
-				f = value.(fs.Fs)
+			// Note that if err == fs.ErrorIsFile at this moment
+			// then we can't rename the remote as it will have the
+			// wrong error status, we need to add a new one.
+			if err == nil {
+				fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", fsString, canonicalName)
+				value, found := c.Rename(fsString, canonicalName)
+				if found {
+					f = value.(fs.Fs)
+				}
+				addMapping(fsString, canonicalName)
+			} else {
+				fs.Debugf(nil, "fs cache: adding new entry for parent of %q, %q", fsString, canonicalName)
+				Put(canonicalName, f)
 			}
-			addMapping(fsString, canonicalName)
 		}
 	}
 	return f, err

fs/cache/cache_test.go

@@ -23,7 +23,7 @@ func mockNewFs(t *testing.T) (func(), func(path string) (fs.Fs, error)) {
 		switch path {
 		case "mock:/":
 			return mockfs.NewFs("mock", "/"), nil
-		case "mock:/file.txt":
+		case "mock:/file.txt", "mock:file.txt":
 			return mockfs.NewFs("mock", "/"), fs.ErrorIsFile
 		case "mock:/error":
 			return nil, errSentinel
@@ -64,13 +64,46 @@ func TestGetFile(t *testing.T) {
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 2, c.Entries())
 
 	f2, err := GetFn("mock:/file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f2)
 
 	assert.Equal(t, f, f2)
+
+	// check parent is there too
+	f2, err = GetFn("mock:/", create)
+	require.Nil(t, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
+}
+
+func TestGetFile2(t *testing.T) {
+	cleanup, create := mockNewFs(t)
+	defer cleanup()
+
+	assert.Equal(t, 0, c.Entries())
+
+	f, err := GetFn("mock:file.txt", create)
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.NotNil(t, f)
+
+	assert.Equal(t, 2, c.Entries())
+
+	f2, err := GetFn("mock:file.txt", create)
+	require.Equal(t, fs.ErrorIsFile, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
+
+	// check parent is there too
+	f2, err = GetFn("mock:/", create)
+	require.Nil(t, err)
+	require.NotNil(t, f2)
+
+	assert.Equal(t, f, f2)
 }
 
 func TestGetError(t *testing.T) {

fs/config/config.go

@@ -172,7 +172,7 @@ func makeConfigPath() string {
 	// Check to see if user supplied a --config variable or environment
 	// variable.  We can't use pflag for this because it isn't initialised
 	// yet so we search the command line manually.
-	_, configSupplied := fs.LookupEnv("RCLONE_CONFIG")
+	_, configSupplied := os.LookupEnv("RCLONE_CONFIG")
 	if !configSupplied {
 		for _, item := range os.Args {
 			if item == "--config" || strings.HasPrefix(item, "--config=") {
@@ -311,7 +311,7 @@ func loadConfigFile() (*goconfig.ConfigFile, error) {
 		} else {
 			usingPasswordCommand = false
 
-			envpw := fs.Getenv("RCLONE_CONFIG_PASS")
+			envpw := os.Getenv("RCLONE_CONFIG_PASS")
 
 			if envpw != "" {
 				err := setConfigPassword(envpw)
@@ -1432,7 +1432,7 @@ func FileGetFlag(section, key string) (string, bool) {
 // It looks up defaults in the environment if they are present
 func FileGet(section, key string, defaultVal ...string) string {
 	envKey := fs.ConfigToEnv(section, key)
-	newValue, found := fs.LookupEnv(envKey)
+	newValue, found := os.LookupEnv(envKey)
 	if found {
 		defaultVal = []string{newValue}
 	}

fs/config/flags/flags.go

@@ -4,6 +4,7 @@ package flags
 
 import (
 	"log"
+	"os"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -14,7 +15,7 @@ import (
 // sets the default from the environment if possible.
 func setDefaultFromEnv(flags *pflag.FlagSet, name string) {
 	key := fs.OptionToEnv(name)
-	newValue, found := fs.LookupEnv(key)
+	newValue, found := os.LookupEnv(key)
 	if found {
 		flag := flags.Lookup(name)
 		if flag == nil {

fs/fingerprint.go

@@ -0,0 +1,54 @@
+package fs
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Fingerprint produces a unique-ish string for an object.
+//
+// This is for detecting whether an object has changed since we last
+// saw it, not for checking object identity between two different
+// remotes - operations.Equal should be used for that.
+//
+// If fast is set then Fingerprint will only include attributes where
+// usually another operation is not required to fetch them. For
+// example if fast is set then this won't include hashes on the local
+// backend.
+func Fingerprint(ctx context.Context, o ObjectInfo, fast bool) string {
+	var (
+		out      strings.Builder
+		f        = o.Fs()
+		features = f.Features()
+	)
+	fmt.Fprintf(&out, "%d", o.Size())
+	// Whether we want to do a slow operation or not
+	//
+	//  fast     true  false true  false
+	//  opIsSlow true  true  false false
+	//  do Op    false true  true  true
+	//
+	// If !fast (slow) do the operation or if !OpIsSlow ==
+	// OpIsFast do the operation.
+	//
+	// Eg don't do this for S3 where modtimes are expensive
+	if !fast || !features.SlowModTime {
+		if f.Precision() != ModTimeNotSupported {
+			fmt.Fprintf(&out, ",%v", o.ModTime(ctx).UTC())
+		}
+	}
+	// Eg don't do this for SFTP/local where hashes are expensive?
+	if !fast || !features.SlowHash {
+		hashType := f.Hashes().GetOne()
+		if hashType != hash.None {
+			hash, err := o.Hash(ctx, hashType)
+			if err == nil {
+				fmt.Fprintf(&out, ",%v", hash)
+			}
+		}
+	}
+	return out.String()
+}

fs/fingerprint_test.go

@@ -0,0 +1,43 @@
+package fs_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fstest/mockfs"
+	"github.com/rclone/rclone/fstest/mockobject"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFingerprint(t *testing.T) {
+	ctx := context.Background()
+	f := mockfs.NewFs("test", "root")
+	f.SetHashes(hash.NewHashSet(hash.MD5))
+
+	for i, test := range []struct {
+		fast        bool
+		slowModTime bool
+		slowHash    bool
+		want        string
+	}{
+		{fast: false, slowModTime: false, slowHash: false, want: "4,0001-01-01 00:00:00 +0000 UTC,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: false, slowModTime: false, slowHash: true, want: "4,0001-01-01 00:00:00 +0000 UTC,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: false, slowModTime: true, slowHash: false, want: "4,0001-01-01 00:00:00 +0000 UTC,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: false, slowModTime: true, slowHash: true, want: "4,0001-01-01 00:00:00 +0000 UTC,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: true, slowModTime: false, slowHash: false, want: "4,0001-01-01 00:00:00 +0000 UTC,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: true, slowModTime: false, slowHash: true, want: "4,0001-01-01 00:00:00 +0000 UTC"},
+		{fast: true, slowModTime: true, slowHash: false, want: "4,8d777f385d3dfec8815d20f7496026dc"},
+		{fast: true, slowModTime: true, slowHash: true, want: "4"},
+	} {
+		what := fmt.Sprintf("#%d fast=%v,slowModTime=%v,slowHash=%v", i, test.fast, test.slowModTime, test.slowHash)
+		o := mockobject.New("potato").WithContent([]byte("data"), mockobject.SeekModeRegular)
+		o.SetFs(f)
+		f.Features().SlowModTime = test.slowModTime
+		f.Features().SlowHash = test.slowHash
+		got := fs.Fingerprint(ctx, o, test.fast)
+		assert.Equal(t, test.want, got, what)
+	}
+}

fs/fs.go

@@ -515,6 +515,8 @@ type Features struct {
 	GetTier                 bool // allows to retrieve storage tier of objects
 	ServerSideAcrossConfigs bool // can server side copy between different remotes of the same type
 	IsLocal                 bool // is the local backend
+	SlowModTime             bool // if calling ModTime() generally takes an extra transaction
+	SlowHash                bool // if calling Hash() generally takes an extra transaction
 
 	// Purge all files in the root and the root directory
 	//
@@ -792,6 +794,10 @@ func (ft *Features) Mask(f Fs) *Features {
 	ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK
 	ft.SetTier = ft.SetTier && mask.SetTier
 	ft.GetTier = ft.GetTier && mask.GetTier
+	ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs
+	// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
+	ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
+	ft.SlowHash = ft.SlowHash && mask.SlowHash
 
 	if mask.Purge == nil {
 		ft.Purge = nil
@@ -1193,25 +1199,9 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err e
 // A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name
 type configEnvVars string
 
-// LookupEnv calls os.LookupEnv and Debugfs a message if the env var was found
-func LookupEnv(key string) (value string, ok bool) {
-	value, ok = os.LookupEnv(key)
-	if ok {
-		Debugf(nil, "Read env var %s", key)
-	}
-	return value, ok
-}
-
-// Getenv calls os.LookupEnv and Debugfs a message if the env var was
-// found. If the var wasn't round it returns empty string.
-func Getenv(key string) (value string) {
-	value, _ = LookupEnv(key)
-	return value
-}
-
 // Get a config item from the environment variables if possible
 func (configName configEnvVars) Get(key string) (value string, ok bool) {
-	return LookupEnv(ConfigToEnv(string(configName), key))
+	return os.LookupEnv(ConfigToEnv(string(configName), key))
 }
 
 // A configmap.Getter to read from the environment RCLONE_option_name
@@ -1227,12 +1217,12 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {
 	}
 	// For options with NoPrefix set, check without prefix too
 	if opt.NoPrefix {
-		value, ok = LookupEnv(OptionToEnv(key))
+		value, ok = os.LookupEnv(OptionToEnv(key))
 		if ok {
 			return value, ok
 		}
 	}
-	return LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
+	return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
 }
 
 // A configmap.Getter to read either the default value or the set

fs/log/log.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/sirupsen/logrus"
 )
 
 // Options contains options for the remote control server
@@ -120,6 +121,7 @@ func InitLogging() {
 			fs.Errorf(nil, "Failed to seek log file to end: %v", err)
 		}
 		log.SetOutput(f)
+		logrus.SetOutput(f)
 		redirectStderr(f)
 	}
 

fs/operations/check.go

@@ -0,0 +1,350 @@
+package operations
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/march"
+	"github.com/rclone/rclone/lib/readers"
+)
+
+// checkFn is the type of the checking function used in CheckFn()
+//
+// It should check the two objects (a, b) and return if they differ
+// and whether the hash was used.
+//
+// If there are differences then this should Errorf the difference and
+// the reason but return with err = nil. It should not CountError in
+// this case.
+type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error)
+
+// CheckOpt contains options for the Check functions
+type CheckOpt struct {
+	Fdst, Fsrc   fs.Fs     // fses to check
+	Check        checkFn   // function to use for checking
+	OneWay       bool      // one way only?
+	Combined     io.Writer // a file with file names with leading sigils
+	MissingOnSrc io.Writer // files only in the destination
+	MissingOnDst io.Writer // files only in the source
+	Match        io.Writer // matching files
+	Differ       io.Writer // differing files
+	Error        io.Writer // files with errors of some kind
+}
+
+// checkMarch is used to march over two Fses in the same way as
+// sync/copy
+type checkMarch struct {
+	ioMu            sync.Mutex
+	wg              sync.WaitGroup
+	tokens          chan struct{}
+	differences     int32
+	noHashes        int32
+	srcFilesMissing int32
+	dstFilesMissing int32
+	matches         int32
+	opt             CheckOpt
+}
+
+// report outputs the fileName to out if required and to the combined log
+func (c *checkMarch) report(o fs.DirEntry, out io.Writer, sigil rune) {
+	if out != nil {
+		c.ioMu.Lock()
+		_, _ = fmt.Fprintf(out, "%v\n", o)
+		c.ioMu.Unlock()
+	}
+	if c.opt.Combined != nil {
+		c.ioMu.Lock()
+		_, _ = fmt.Fprintf(c.opt.Combined, "%c %v\n", sigil, o)
+		c.ioMu.Unlock()
+	}
+}
+
+// DstOnly have an object which is in the destination only
+func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
+	switch dst.(type) {
+	case fs.Object:
+		if c.opt.OneWay {
+			return false
+		}
+		err := errors.Errorf("File not in %v", c.opt.Fsrc)
+		fs.Errorf(dst, "%v", err)
+		_ = fs.CountError(err)
+		atomic.AddInt32(&c.differences, 1)
+		atomic.AddInt32(&c.srcFilesMissing, 1)
+		c.report(dst, c.opt.MissingOnSrc, '-')
+	case fs.Directory:
+		// Do the same thing to the entire contents of the directory
+		if c.opt.OneWay {
+			return false
+		}
+		return true
+	default:
+		panic("Bad object in DirEntries")
+	}
+	return false
+}
+
+// SrcOnly have an object which is in the source only
+func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
+	switch src.(type) {
+	case fs.Object:
+		err := errors.Errorf("File not in %v", c.opt.Fdst)
+		fs.Errorf(src, "%v", err)
+		_ = fs.CountError(err)
+		atomic.AddInt32(&c.differences, 1)
+		atomic.AddInt32(&c.dstFilesMissing, 1)
+		c.report(src, c.opt.MissingOnDst, '+')
+	case fs.Directory:
+		// Do the same thing to the entire contents of the directory
+		return true
+	default:
+		panic("Bad object in DirEntries")
+	}
+	return false
+}
+
+// check to see if two objects are identical using the check function
+func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+	tr := accounting.Stats(ctx).NewCheckingTransfer(src)
+	defer func() {
+		tr.Done(err)
+	}()
+	if sizeDiffers(src, dst) {
+		err = errors.Errorf("Sizes differ")
+		fs.Errorf(src, "%v", err)
+		return true, false, nil
+	}
+	if fs.Config.SizeOnly {
+		return false, false, nil
+	}
+	return c.opt.Check(ctx, dst, src)
+}
+
+// Match is called when src and dst are present, so sync src to dst
+func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
+	switch srcX := src.(type) {
+	case fs.Object:
+		dstX, ok := dst.(fs.Object)
+		if ok {
+			if SkipDestructive(ctx, src, "check") {
+				return false
+			}
+			c.wg.Add(1)
+			c.tokens <- struct{}{} // put a token to limit concurrency
+			go func() {
+				defer func() {
+					<-c.tokens // get the token back to free up a slot
+					c.wg.Done()
+				}()
+				differ, noHash, err := c.checkIdentical(ctx, dstX, srcX)
+				if err != nil {
+					fs.Errorf(src, "%v", err)
+					_ = fs.CountError(err)
+					c.report(src, c.opt.Error, '!')
+				} else if differ {
+					atomic.AddInt32(&c.differences, 1)
+					err := errors.New("files differ")
+					// the checkFn has already logged the reason
+					_ = fs.CountError(err)
+					c.report(src, c.opt.Differ, '*')
+				} else {
+					atomic.AddInt32(&c.matches, 1)
+					c.report(src, c.opt.Match, '=')
+					if noHash {
+						atomic.AddInt32(&c.noHashes, 1)
+						fs.Debugf(dstX, "OK - could not check hash")
+					} else {
+						fs.Debugf(dstX, "OK")
+					}
+				}
+			}()
+		} else {
+			err := errors.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst)
+			fs.Errorf(src, "%v", err)
+			_ = fs.CountError(err)
+			atomic.AddInt32(&c.differences, 1)
+			atomic.AddInt32(&c.dstFilesMissing, 1)
+			c.report(src, c.opt.MissingOnDst, '+')
+		}
+	case fs.Directory:
+		// Do the same thing to the entire contents of the directory
+		_, ok := dst.(fs.Directory)
+		if ok {
+			return true
+		}
+		err := errors.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
+		fs.Errorf(dst, "%v", err)
+		_ = fs.CountError(err)
+		atomic.AddInt32(&c.differences, 1)
+		atomic.AddInt32(&c.srcFilesMissing, 1)
+		c.report(dst, c.opt.MissingOnSrc, '-')
+
+	default:
+		panic("Bad object in DirEntries")
+	}
+	return false
+}
+
+// CheckFn checks the files in fsrc and fdst according to Size and
+// hash using checkFunction on each file to check the hashes.
+//
+// checkFunction sees if dst and src are identical
+//
+// it returns true if differences were found
+// it also returns whether it couldn't be hashed
+func CheckFn(ctx context.Context, opt *CheckOpt) error {
+	if opt.Check == nil {
+		return errors.New("internal error: nil check function")
+	}
+	c := &checkMarch{
+		tokens: make(chan struct{}, fs.Config.Checkers),
+		opt:    *opt,
+	}
+
+	// set up a march over fdst and fsrc
+	m := &march.March{
+		Ctx:      ctx,
+		Fdst:     c.opt.Fdst,
+		Fsrc:     c.opt.Fsrc,
+		Dir:      "",
+		Callback: c,
+	}
+	fs.Debugf(c.opt.Fdst, "Waiting for checks to finish")
+	err := m.Run()
+	c.wg.Wait() // wait for background go-routines
+
+	if c.dstFilesMissing > 0 {
+		fs.Logf(c.opt.Fdst, "%d files missing", c.dstFilesMissing)
+	}
+	if c.srcFilesMissing > 0 {
+		fs.Logf(c.opt.Fsrc, "%d files missing", c.srcFilesMissing)
+	}
+
+	fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
+	if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
+		fs.Logf(c.opt.Fdst, "%d errors while checking", errs)
+	}
+	if c.noHashes > 0 {
+		fs.Logf(c.opt.Fdst, "%d hashes could not be checked", c.noHashes)
+	}
+	if c.matches > 0 {
+		fs.Logf(c.opt.Fdst, "%d matching files", c.matches)
+	}
+	if c.differences > 0 {
+		return errors.Errorf("%d differences found", c.differences)
+	}
+	return err
+}
+
+// Check the files in fsrc and fdst according to Size and hash
+func Check(ctx context.Context, opt *CheckOpt) error {
+	optCopy := *opt
+	optCopy.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+		same, ht, err := CheckHashes(ctx, src, dst)
+		if err != nil {
+			return true, false, err
+		}
+		if ht == hash.None {
+			return false, true, nil
+		}
+		if !same {
+			err = errors.Errorf("%v differ", ht)
+			fs.Errorf(src, "%v", err)
+			return true, false, nil
+		}
+		return false, false, nil
+	}
+
+	return CheckFn(ctx, &optCopy)
+}
+
+// CheckEqualReaders checks to see if in1 and in2 have the same
+// content when read.
+//
+// it returns true if differences were found
+func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) {
+	const bufSize = 64 * 1024
+	buf1 := make([]byte, bufSize)
+	buf2 := make([]byte, bufSize)
+	for {
+		n1, err1 := readers.ReadFill(in1, buf1)
+		n2, err2 := readers.ReadFill(in2, buf2)
+		// check errors
+		if err1 != nil && err1 != io.EOF {
+			return true, err1
+		} else if err2 != nil && err2 != io.EOF {
+			return true, err2
+		}
+		// err1 && err2 are nil or io.EOF here
+		// process the data
+		if n1 != n2 || !bytes.Equal(buf1[:n1], buf2[:n2]) {
+			return true, nil
+		}
+		// if both streams finished the we have finished
+		if err1 == io.EOF && err2 == io.EOF {
+			break
+		}
+	}
+	return false, nil
+}
+
+// CheckIdenticalDownload checks to see if dst and src are identical
+// by reading all their bytes if necessary.
+//
+// it returns true if differences were found
+func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
+	err = Retry(src, fs.Config.LowLevelRetries, func() error {
+		differ, err = checkIdenticalDownload(ctx, dst, src)
+		return err
+	})
+	return differ, err
+}
+
+// Does the work for CheckIdenticalDownload
+func checkIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
+	in1, err := dst.Open(ctx)
+	if err != nil {
+		return true, errors.Wrapf(err, "failed to open %q", dst)
+	}
+	tr1 := accounting.Stats(ctx).NewTransfer(dst)
+	defer func() {
+		tr1.Done(nil) // error handling is done by the caller
+	}()
+	in1 = tr1.Account(in1).WithBuffer() // account and buffer the transfer
+
+	in2, err := src.Open(ctx)
+	if err != nil {
+		return true, errors.Wrapf(err, "failed to open %q", src)
+	}
+	tr2 := accounting.Stats(ctx).NewTransfer(dst)
+	defer func() {
+		tr2.Done(nil) // error handling is done by the caller
+	}()
+	in2 = tr2.Account(in2).WithBuffer() // account and buffer the transfer
+
+	// To assign err variable before defer.
+	differ, err = CheckEqualReaders(in1, in2)
+	return
+}
+
+// CheckDownload checks the files in fsrc and fdst according to Size
+// and the actual contents of the files.
+func CheckDownload(ctx context.Context, opt *CheckOpt) error {
+	optCopy := *opt
+	optCopy.Check = func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error) {
+		differ, err = CheckIdenticalDownload(ctx, a, b)
+		if err != nil {
+			return true, true, errors.Wrap(err, "failed to download")
+		}
+		return differ, false, nil
+	}
+	return CheckFn(ctx, &optCopy)
+}

fs/operations/check_test.go

@@ -0,0 +1,272 @@
+package operations_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	addBuffers := func(opt *operations.CheckOpt) {
+		opt.Combined = new(bytes.Buffer)
+		opt.MissingOnSrc = new(bytes.Buffer)
+		opt.MissingOnDst = new(bytes.Buffer)
+		opt.Match = new(bytes.Buffer)
+		opt.Differ = new(bytes.Buffer)
+		opt.Error = new(bytes.Buffer)
+	}
+
+	sortLines := func(in string) []string {
+		if in == "" {
+			return []string{}
+		}
+		lines := strings.Split(in, "\n")
+		sort.Strings(lines)
+		return lines
+	}
+
+	checkBuffer := func(name string, want map[string]string, out io.Writer) {
+		expected := want[name]
+		buf, ok := out.(*bytes.Buffer)
+		require.True(t, ok)
+		assert.Equal(t, sortLines(expected), sortLines(buf.String()), name)
+	}
+
+	checkBuffers := func(opt *operations.CheckOpt, want map[string]string) {
+		checkBuffer("combined", want, opt.Combined)
+		checkBuffer("missingonsrc", want, opt.MissingOnSrc)
+		checkBuffer("missingondst", want, opt.MissingOnDst)
+		checkBuffer("match", want, opt.Match)
+		checkBuffer("differ", want, opt.Differ)
+		checkBuffer("error", want, opt.Error)
+	}
+
+	check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			accounting.GlobalStats().ResetCounters()
+			var buf bytes.Buffer
+			log.SetOutput(&buf)
+			defer func() {
+				log.SetOutput(os.Stderr)
+			}()
+			opt := operations.CheckOpt{
+				Fdst:   r.Fremote,
+				Fsrc:   r.Flocal,
+				OneWay: oneway,
+			}
+			addBuffers(&opt)
+			err := checkFunction(context.Background(), &opt)
+			gotErrors := accounting.GlobalStats().GetErrors()
+			gotChecks := accounting.GlobalStats().GetChecks()
+			if wantErrors == 0 && err != nil {
+				t.Errorf("%d: Got error when not expecting one: %v", i, err)
+			}
+			if wantErrors != 0 && err == nil {
+				t.Errorf("%d: No error when expecting one", i)
+			}
+			if wantErrors != gotErrors {
+				t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
+			}
+			if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
+				t.Errorf("%d: Total files matching line missing", i)
+			}
+			if wantChecks != gotChecks {
+				t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
+			}
+			checkBuffers(&opt, wantOutput)
+		})
+	}
+
+	file1 := r.WriteBoth(context.Background(), "rutabaga", "is tasty", t3)
+	fstest.CheckItems(t, r.Fremote, file1)
+	fstest.CheckItems(t, r.Flocal, file1)
+	check(1, 0, 1, false, map[string]string{
+		"combined":     "= rutabaga\n",
+		"missingonsrc": "",
+		"missingondst": "",
+		"match":        "rutabaga\n",
+		"differ":       "",
+		"error":        "",
+	})
+
+	file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
+	fstest.CheckItems(t, r.Flocal, file1, file2)
+	check(2, 1, 1, false, map[string]string{
+		"combined":     "+ potato2\n= rutabaga\n",
+		"missingonsrc": "",
+		"missingondst": "potato2\n",
+		"match":        "rutabaga\n",
+		"differ":       "",
+		"error":        "",
+	})
+
+	file3 := r.WriteObject(context.Background(), "empty space", "-", t2)
+	fstest.CheckItems(t, r.Fremote, file1, file3)
+	check(3, 2, 1, false, map[string]string{
+		"combined":     "- empty space\n+ potato2\n= rutabaga\n",
+		"missingonsrc": "empty space\n",
+		"missingondst": "potato2\n",
+		"match":        "rutabaga\n",
+		"differ":       "",
+		"error":        "",
+	})
+
+	file2r := file2
+	if fs.Config.SizeOnly {
+		file2r = r.WriteObject(context.Background(), "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
+	} else {
+		r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
+	}
+	fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
+	check(4, 1, 2, false, map[string]string{
+		"combined":     "- empty space\n= potato2\n= rutabaga\n",
+		"missingonsrc": "empty space\n",
+		"missingondst": "",
+		"match":        "rutabaga\npotato2\n",
+		"differ":       "",
+		"error":        "",
+	})
+
+	file3r := file3
+	file3l := r.WriteFile("empty space", "DIFFER", t2)
+	fstest.CheckItems(t, r.Flocal, file1, file2, file3l)
+	check(5, 1, 3, false, map[string]string{
+		"combined":     "* empty space\n= potato2\n= rutabaga\n",
+		"missingonsrc": "",
+		"missingondst": "",
+		"match":        "potato2\nrutabaga\n",
+		"differ":       "empty space\n",
+		"error":        "",
+	})
+
+	file4 := r.WriteObject(context.Background(), "remotepotato", "------------------------------------------------------------", t1)
+	fstest.CheckItems(t, r.Fremote, file1, file2r, file3r, file4)
+	check(6, 2, 3, false, map[string]string{
+		"combined":     "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
+		"missingonsrc": "remotepotato\n",
+		"missingondst": "",
+		"match":        "potato2\nrutabaga\n",
+		"differ":       "empty space\n",
+		"error":        "",
+	})
+	check(7, 1, 3, true, map[string]string{
+		"combined":     "* empty space\n= potato2\n= rutabaga\n",
+		"missingonsrc": "",
+		"missingondst": "",
+		"match":        "potato2\nrutabaga\n",
+		"differ":       "empty space\n",
+		"error":        "",
+	})
+}
+
+func TestCheck(t *testing.T) {
+	testCheck(t, operations.Check)
+}
+
+func TestCheckFsError(t *testing.T) {
+	dstFs, err := fs.NewFs("non-existent")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srcFs, err := fs.NewFs("non-existent")
+	if err != nil {
+		t.Fatal(err)
+	}
+	opt := operations.CheckOpt{
+		Fdst:   dstFs,
+		Fsrc:   srcFs,
+		OneWay: false,
+	}
+	err = operations.Check(context.Background(), &opt)
+	require.Error(t, err)
+}
+
+func TestCheckDownload(t *testing.T) {
+	testCheck(t, operations.CheckDownload)
+}
+
+func TestCheckSizeOnly(t *testing.T) {
+	fs.Config.SizeOnly = true
+	defer func() { fs.Config.SizeOnly = false }()
+	TestCheck(t)
+}
+
+func TestCheckEqualReaders(t *testing.T) {
+	b65a := make([]byte, 65*1024)
+	b65b := make([]byte, 65*1024)
+	b65b[len(b65b)-1] = 1
+	b66 := make([]byte, 66*1024)
+
+	differ, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a))
+	assert.NoError(t, err)
+	assert.Equal(t, differ, false)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b))
+	assert.NoError(t, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66))
+	assert.NoError(t, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a))
+	assert.NoError(t, err)
+	assert.Equal(t, differ, true)
+
+	myErr := errors.New("sentinel")
+	wrap := func(b []byte) io.Reader {
+		r := bytes.NewBuffer(b)
+		e := readers.ErrorReader{Err: myErr}
+		return io.MultiReader(r, e)
+	}
+
+	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+
+	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a))
+	assert.Equal(t, myErr, err)
+	assert.Equal(t, differ, true)
+}

fs/operations/operations.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/march"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/atexit"
@@ -726,246 +725,6 @@ func SameDir(fdst, fsrc fs.Info) bool {
 	return fdstRoot == fsrcRoot
 }
 
-// checkIdentical checks to see if dst and src are identical
-//
-// it returns true if differences were found
-// it also returns whether it couldn't be hashed
-func checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
-	same, ht, err := CheckHashes(ctx, src, dst)
-	if err != nil {
-		// CheckHashes will log and count errors
-		return true, false
-	}
-	if ht == hash.None {
-		return false, true
-	}
-	if !same {
-		err = errors.Errorf("%v differ", ht)
-		fs.Errorf(src, "%v", err)
-		_ = fs.CountError(err)
-		return true, false
-	}
-	return false, false
-}
-
-// checkFn is the type of the checking function used in CheckFn()
-type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool)
-
-// checkMarch is used to march over two Fses in the same way as
-// sync/copy
-type checkMarch struct {
-	fdst, fsrc      fs.Fs
-	check           checkFn
-	wg              sync.WaitGroup
-	tokens          chan struct{}
-	oneway          bool
-	differences     int32
-	noHashes        int32
-	srcFilesMissing int32
-	dstFilesMissing int32
-	matches         int32
-}
-
-// DstOnly have an object which is in the destination only
-func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
-	switch dst.(type) {
-	case fs.Object:
-		if c.oneway {
-			return false
-		}
-		err := errors.Errorf("File not in %v", c.fsrc)
-		fs.Errorf(dst, "%v", err)
-		_ = fs.CountError(err)
-		atomic.AddInt32(&c.differences, 1)
-		atomic.AddInt32(&c.srcFilesMissing, 1)
-	case fs.Directory:
-		// Do the same thing to the entire contents of the directory
-		if c.oneway {
-			return false
-		}
-		return true
-	default:
-		panic("Bad object in DirEntries")
-	}
-	return false
-}
-
-// SrcOnly have an object which is in the source only
-func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
-	switch src.(type) {
-	case fs.Object:
-		err := errors.Errorf("File not in %v", c.fdst)
-		fs.Errorf(src, "%v", err)
-		_ = fs.CountError(err)
-		atomic.AddInt32(&c.differences, 1)
-		atomic.AddInt32(&c.dstFilesMissing, 1)
-	case fs.Directory:
-		// Do the same thing to the entire contents of the directory
-		return true
-	default:
-		panic("Bad object in DirEntries")
-	}
-	return false
-}
-
-// check to see if two objects are identical using the check function
-func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
-	var err error
-	tr := accounting.Stats(ctx).NewCheckingTransfer(src)
-	defer func() {
-		tr.Done(err)
-	}()
-	if sizeDiffers(src, dst) {
-		err = errors.Errorf("Sizes differ")
-		fs.Errorf(src, "%v", err)
-		return true, false
-	}
-	if fs.Config.SizeOnly {
-		return false, false
-	}
-	return c.check(ctx, dst, src)
-}
-
-// Match is called when src and dst are present, so sync src to dst
-func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
-	switch srcX := src.(type) {
-	case fs.Object:
-		dstX, ok := dst.(fs.Object)
-		if ok {
-			if SkipDestructive(ctx, src, "check") {
-				return false
-			}
-			c.wg.Add(1)
-			c.tokens <- struct{}{} // put a token to limit concurrency
-			go func() {
-				defer func() {
-					<-c.tokens // get the token back to free up a slot
-					c.wg.Done()
-				}()
-				differ, noHash := c.checkIdentical(ctx, dstX, srcX)
-				if differ {
-					atomic.AddInt32(&c.differences, 1)
-				} else {
-					atomic.AddInt32(&c.matches, 1)
-					if noHash {
-						atomic.AddInt32(&c.noHashes, 1)
-						fs.Debugf(dstX, "OK - could not check hash")
-					} else {
-						fs.Debugf(dstX, "OK")
-					}
-				}
-			}()
-		} else {
-			err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst)
-			fs.Errorf(src, "%v", err)
-			_ = fs.CountError(err)
-			atomic.AddInt32(&c.differences, 1)
-			atomic.AddInt32(&c.dstFilesMissing, 1)
-		}
-	case fs.Directory:
-		// Do the same thing to the entire contents of the directory
-		_, ok := dst.(fs.Directory)
-		if ok {
-			return true
-		}
-		err := errors.Errorf("is file on %v but directory on %v", c.fdst, c.fsrc)
-		fs.Errorf(dst, "%v", err)
-		_ = fs.CountError(err)
-		atomic.AddInt32(&c.differences, 1)
-		atomic.AddInt32(&c.srcFilesMissing, 1)
-
-	default:
-		panic("Bad object in DirEntries")
-	}
-	return false
-}
-
-// CheckFn checks the files in fsrc and fdst according to Size and
-// hash using checkFunction on each file to check the hashes.
-//
-// checkFunction sees if dst and src are identical
-//
-// it returns true if differences were found
-// it also returns whether it couldn't be hashed
-func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
-	c := &checkMarch{
-		fdst:   fdst,
-		fsrc:   fsrc,
-		check:  check,
-		oneway: oneway,
-		tokens: make(chan struct{}, fs.Config.Checkers),
-	}
-
-	// set up a march over fdst and fsrc
-	m := &march.March{
-		Ctx:      ctx,
-		Fdst:     fdst,
-		Fsrc:     fsrc,
-		Dir:      "",
-		Callback: c,
-	}
-	fs.Debugf(fdst, "Waiting for checks to finish")
-	err := m.Run()
-	c.wg.Wait() // wait for background go-routines
-
-	if c.dstFilesMissing > 0 {
-		fs.Logf(fdst, "%d files missing", c.dstFilesMissing)
-	}
-	if c.srcFilesMissing > 0 {
-		fs.Logf(fsrc, "%d files missing", c.srcFilesMissing)
-	}
-
-	fs.Logf(fdst, "%d differences found", c.differences)
-	if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
-		fs.Logf(fdst, "%d errors while checking", errs)
-	}
-	if c.noHashes > 0 {
-		fs.Logf(fdst, "%d hashes could not be checked", c.noHashes)
-	}
-	if c.matches > 0 {
-		fs.Logf(fdst, "%d matching files", c.matches)
-	}
-	if c.differences > 0 {
-		return errors.Errorf("%d differences found", c.differences)
-	}
-	return err
-}
-
-// Check the files in fsrc and fdst according to Size and hash
-func Check(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
-	return CheckFn(ctx, fdst, fsrc, checkIdentical, oneway)
-}
-
-// CheckEqualReaders checks to see if in1 and in2 have the same
-// content when read.
-//
-// it returns true if differences were found
-func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) {
-	const bufSize = 64 * 1024
-	buf1 := make([]byte, bufSize)
-	buf2 := make([]byte, bufSize)
-	for {
-		n1, err1 := readers.ReadFill(in1, buf1)
-		n2, err2 := readers.ReadFill(in2, buf2)
-		// check errors
-		if err1 != nil && err1 != io.EOF {
-			return true, err1
-		} else if err2 != nil && err2 != io.EOF {
-			return true, err2
-		}
-		// err1 && err2 are nil or io.EOF here
-		// process the data
-		if n1 != n2 || !bytes.Equal(buf1[:n1], buf2[:n2]) {
-			return true, nil
-		}
-		// if both streams finished the we have finished
-		if err1 == io.EOF && err2 == io.EOF {
-			break
-		}
-	}
-	return false, nil
-}
-
 // Retry runs fn up to maxTries times if it returns a retriable error
 func Retry(o interface{}, maxTries int, fn func() error) (err error) {
 	for tries := 1; tries <= maxTries; tries++ {
@@ -984,60 +743,6 @@ func Retry(o interface{}, maxTries int, fn func() error) (err error) {
 	return err
 }
 
-// CheckIdenticalDownload checks to see if dst and src are identical
-// by reading all their bytes if necessary.
-//
-// it returns true if differences were found
-func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
-	err = Retry(src, fs.Config.LowLevelRetries, func() error {
-		differ, err = checkIdenticalDownload(ctx, dst, src)
-		return err
-	})
-	return differ, err
-}
-
-// Does the work for CheckIdenticalDownload
-func checkIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
-	in1, err := dst.Open(ctx)
-	if err != nil {
-		return true, errors.Wrapf(err, "failed to open %q", dst)
-	}
-	tr1 := accounting.Stats(ctx).NewTransfer(dst)
-	defer func() {
-		tr1.Done(nil) // error handling is done by the caller
-	}()
-	in1 = tr1.Account(in1).WithBuffer() // account and buffer the transfer
-
-	in2, err := src.Open(ctx)
-	if err != nil {
-		return true, errors.Wrapf(err, "failed to open %q", src)
-	}
-	tr2 := accounting.Stats(ctx).NewTransfer(dst)
-	defer func() {
-		tr2.Done(nil) // error handling is done by the caller
-	}()
-	in2 = tr2.Account(in2).WithBuffer() // account and buffer the transfer
-
-	// To assign err variable before defer.
-	differ, err = CheckEqualReaders(in1, in2)
-	return
-}
-
-// CheckDownload checks the files in fsrc and fdst according to Size
-// and the actual contents of the files.
-func CheckDownload(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
-	check := func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool) {
-		differ, err := CheckIdenticalDownload(ctx, a, b)
-		if err != nil {
-			err = fs.CountError(err)
-			fs.Errorf(a, "Failed to download: %v", err)
-			return true, true
-		}
-		return differ, false
-	}
-	return CheckFn(ctx, fdst, fsrc, check, oneway)
-}
-
 // ListFn lists the Fs to the supplied function
 //
 // Lists in parallel which may get them out of order

fs/operations/operations_test.go

@@ -22,11 +22,9 @@ package operations_test
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -45,7 +43,6 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -354,98 +351,6 @@ func TestRetry(t *testing.T) {
 
 }
 
-func testCheck(t *testing.T, checkFunction func(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error) {
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-
-	check := func(i int, wantErrors int64, wantChecks int64, oneway bool) {
-		fs.Debugf(r.Fremote, "%d: Starting check test", i)
-		accounting.GlobalStats().ResetCounters()
-		var buf bytes.Buffer
-		log.SetOutput(&buf)
-		defer func() {
-			log.SetOutput(os.Stderr)
-		}()
-		err := checkFunction(context.Background(), r.Fremote, r.Flocal, oneway)
-		gotErrors := accounting.GlobalStats().GetErrors()
-		gotChecks := accounting.GlobalStats().GetChecks()
-		if wantErrors == 0 && err != nil {
-			t.Errorf("%d: Got error when not expecting one: %v", i, err)
-		}
-		if wantErrors != 0 && err == nil {
-			t.Errorf("%d: No error when expecting one", i)
-		}
-		if wantErrors != gotErrors {
-			t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
-		}
-		if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
-			t.Errorf("%d: Total files matching line missing", i)
-		}
-		if wantChecks != gotChecks {
-			t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
-		}
-		fs.Debugf(r.Fremote, "%d: Ending check test", i)
-	}
-
-	file1 := r.WriteBoth(context.Background(), "rutabaga", "is tasty", t3)
-	fstest.CheckItems(t, r.Fremote, file1)
-	fstest.CheckItems(t, r.Flocal, file1)
-	check(1, 0, 1, false)
-
-	file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
-	fstest.CheckItems(t, r.Flocal, file1, file2)
-	check(2, 1, 1, false)
-
-	file3 := r.WriteObject(context.Background(), "empty space", "-", t2)
-	fstest.CheckItems(t, r.Fremote, file1, file3)
-	check(3, 2, 1, false)
-
-	file2r := file2
-	if fs.Config.SizeOnly {
-		file2r = r.WriteObject(context.Background(), "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
-	} else {
-		r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
-	}
-	fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
-	check(4, 1, 2, false)
-
-	r.WriteFile("empty space", "-", t2)
-	fstest.CheckItems(t, r.Flocal, file1, file2, file3)
-	check(5, 0, 3, false)
-
-	file4 := r.WriteObject(context.Background(), "remotepotato", "------------------------------------------------------------", t1)
-	fstest.CheckItems(t, r.Fremote, file1, file2r, file3, file4)
-	check(6, 1, 3, false)
-	check(7, 0, 3, true)
-}
-
-func TestCheck(t *testing.T) {
-	testCheck(t, operations.Check)
-}
-
-func TestCheckFsError(t *testing.T) {
-	dstFs, err := fs.NewFs("non-existent")
-	if err != nil {
-		t.Fatal(err)
-	}
-	srcFs, err := fs.NewFs("non-existent")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = operations.Check(context.Background(), dstFs, srcFs, false)
-	require.Error(t, err)
-}
-
-func TestCheckDownload(t *testing.T) {
-	testCheck(t, operations.CheckDownload)
-}
-
-func TestCheckSizeOnly(t *testing.T) {
-	fs.Config.SizeOnly = true
-	defer func() { fs.Config.SizeOnly = false }()
-	TestCheck(t)
-}
-
 func TestCat(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
@@ -1179,68 +1084,6 @@ func TestOverlapping(t *testing.T) {
 	}
 }
 
-func TestCheckEqualReaders(t *testing.T) {
-	b65a := make([]byte, 65*1024)
-	b65b := make([]byte, 65*1024)
-	b65b[len(b65b)-1] = 1
-	b66 := make([]byte, 66*1024)
-
-	differ, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a))
-	assert.NoError(t, err)
-	assert.Equal(t, differ, false)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b))
-	assert.NoError(t, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66))
-	assert.NoError(t, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a))
-	assert.NoError(t, err)
-	assert.Equal(t, differ, true)
-
-	myErr := errors.New("sentinel")
-	wrap := func(b []byte) io.Reader {
-		r := bytes.NewBuffer(b)
-		e := readers.ErrorReader{Err: myErr}
-		return io.MultiReader(r, e)
-	}
-
-	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-
-	differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a))
-	assert.Equal(t, myErr, err)
-	assert.Equal(t, differ, true)
-}
-
 func TestListFormat(t *testing.T) {
 	item0 := &operations.ListJSONItem{
 		Path:      "a",

fs/sync/sync.go

@@ -658,6 +658,7 @@ func (s *syncCopyMove) pushRenameMap(hash string, obj fs.Object) {
 // renameMap or returns nil if not found.
 func (s *syncCopyMove) popRenameMap(hash string, src fs.Object) (dst fs.Object) {
 	s.renameMapMu.Lock()
+	defer s.renameMapMu.Unlock()
 	dsts, ok := s.renameMap[hash]
 	if ok && len(dsts) > 0 {
 		// Element to remove
@@ -690,7 +691,6 @@ func (s *syncCopyMove) popRenameMap(hash string, src fs.Object) (dst fs.Object)
 			delete(s.renameMap, hash)
 		}
 	}
-	s.renameMapMu.Unlock()
 	return dst
 }
 

fstest/mockfs/mockfs.go

@@ -18,6 +18,7 @@ type Fs struct {
 	root     string        // The root directory (OS path)
 	features *fs.Features  // optional features
 	rootDir  fs.DirEntries // directory listing of root
+	hashes   hash.Set      // which hashes we support
 }
 
 // ErrNotImplemented is returned by unimplemented methods
@@ -66,7 +67,12 @@ func (f *Fs) Precision() time.Duration {
 
 // Hashes returns the supported hash types of the filesystem
 func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet()
+	return f.hashes
+}
+
+// SetHashes sets the hashes that this supports
+func (f *Fs) SetHashes(hashes hash.Set) {
+	f.hashes = hashes
 }
 
 // Features returns the optional features of this Fs

fstest/mockobject/mockobject.go

@@ -178,6 +178,20 @@ func (o *ContentMockObject) Size() int64 {
 	return int64(len(o.content))
 }
 
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *ContentMockObject) Hash(ctx context.Context, t hash.Type) (string, error) {
+	hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(t))
+	if err != nil {
+		return "", err
+	}
+	_, err = hasher.Write(o.content)
+	if err != nil {
+		return "", err
+	}
+	return hasher.Sums()[t], nil
+}
+
 type readCloser struct{ io.Reader }
 
 func (r *readCloser) Close() error { return nil }

go.mod

@@ -1,73 +1,73 @@
 module github.com/rclone/rclone
 
+go 1.14
+
 require (
-	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc
-	cloud.google.com/go v0.53.0 // indirect
+	bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512
+	cloud.google.com/go v0.59.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2
-	github.com/Azure/azure-storage-blob-go v0.8.0
-	github.com/Azure/go-autorest/autorest/adal v0.6.0 // indirect
+	github.com/Azure/azure-storage-blob-go v0.10.0
 	github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
 	github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4
-	github.com/aalpar/deheap v0.0.0-20191229192855-f837f7a9ba26
+	github.com/aalpar/deheap v0.0.0-20200318053559-9a0c2883bd56
 	github.com/abbot/go-http-auth v0.4.0
 	github.com/anacrolix/dms v1.1.0
 	github.com/atotto/clipboard v0.1.2
-	github.com/aws/aws-sdk-go v1.29.9
-	github.com/billziss-gh/cgofuse v1.2.0
+	github.com/aws/aws-sdk-go v1.32.11
+	github.com/billziss-gh/cgofuse v1.3.0
+	github.com/btcsuite/btcutil v1.0.2 // indirect
+	github.com/calebcase/tmpfile v1.0.2 // indirect
 	github.com/coreos/go-semver v0.3.0
-	github.com/djherbis/times v1.2.0
 	github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible
+	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
-	github.com/hanwen/go-fuse/v2 v2.0.3-0.20191108143333-152e6ac32d54
-	github.com/jlaffaye/ftp v0.0.0-20200422224957-b9f3ade29122
+	github.com/hanwen/go-fuse/v2 v2.0.3
+	github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e
 	github.com/jzelinskie/whirlpool v0.0.0-20170603002051-c19460b8caa6
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc
+	github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348
 	github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a
-	github.com/mattn/go-colorable v0.1.4
-	github.com/mattn/go-ieproxy v0.0.0-20200203040449-2dbc853185d9 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.8
+	github.com/mattn/go-colorable v0.1.7
+	github.com/mattn/go-ieproxy v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.9
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2
-	github.com/ncw/swift v1.0.50
-	github.com/nsf/termbox-go v0.0.0-20200204031403-4d2b513ad8be
+	github.com/ncw/swift v1.0.52
+	github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1
 	github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
-	github.com/onsi/ginkgo v1.9.0 // indirect
-	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.11.0
-	github.com/prometheus/client_golang v1.4.1
+	github.com/prometheus/client_golang v1.7.1
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
-	github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46
+	github.com/rfjakob/eme v1.1.1
 	github.com/sevlyar/go-daemon v0.1.5
-	github.com/sirupsen/logrus v1.4.2
+	github.com/sirupsen/logrus v1.6.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
-	github.com/smartystreets/assertions v1.0.1 // indirect
-	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.5.1
-	github.com/t3rm1n4l/go-mega v0.0.0-20200117211730-79a813bb328d
+	github.com/stretchr/testify v1.6.1
+	github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8
 	github.com/xanzy/ssh-agent v0.2.1
-	github.com/youmark/pkcs8 v0.0.0-20191102193632-94c173a94d60
+	github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0
-	go.etcd.io/bbolt v1.3.3
-	goftp.io/server v0.3.2
-	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
-	golang.org/x/net v0.0.0-20200222125558-5a598a2470a0
+	go.etcd.io/bbolt v1.3.5
+	go.opencensus.io v0.22.4 // indirect
+	go.uber.org/zap v1.15.0 // indirect
+	goftp.io/server v0.4.0
+	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
-	golang.org/x/text v0.3.2
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	google.golang.org/api v0.21.1-0.20200411000818-c8cf5cff125e
-	google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688 // indirect
-	gopkg.in/yaml.v2 v2.2.8
-	storj.io/uplink v1.1.1
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666
+	golang.org/x/text v0.3.3
+	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
+	google.golang.org/api v0.28.0
+	google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5 // indirect
+	google.golang.org/grpc v1.30.0 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+	storj.io/common v0.0.0-20200624215549-bf610d22d466 // indirect
+	storj.io/uplink v1.1.2
 )
-
-go 1.14

go.sum

@@ -1,5 +1,5 @@
-bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc h1:utDghgcjE8u+EBjHOgYT+dJPcnDF05KqWMBcjuJy510=
-bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
+bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 h1:SRsZGA7aFnCZETmov57jwPrWuTmaZK6+4R4v5FUe1/c=
+bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -8,26 +8,40 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.59.0 h1:BM3svUDU3itpc2m5cu5wCyThIYNDlFlts9GASw31GW8=
+cloud.google.com/go v0.59.0/go.mod h1:qJxNOVCRTxHfwLhvDxxSI9vQc1zI59b9pEglp1Iv60E=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
 github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
-github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
-github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
+github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs=
+github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE=
 github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo=
-github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/adal v0.8.3 h1:O1AGG9Xig71FxdX9HO5pGNyZ7TbSyHaVg+5eJO/jSGw=
+github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
 github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
 github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
@@ -48,8 +62,8 @@ github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBk
 github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4 h1:mK1/QgFPU4osbhjJ26B1w738kjQHaGJcon8uCLMS8fk=
 github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
-github.com/aalpar/deheap v0.0.0-20191229192855-f837f7a9ba26 h1:vn0bf9tSbwrl9aBeG5hqGPpWeZGEybrB3LUJkHP/x6w=
-github.com/aalpar/deheap v0.0.0-20191229192855-f837f7a9ba26/go.mod h1:EJFoWbcEEVK22GYKONJjtMNamGYe6p+3x1Pr6zV5gFs=
+github.com/aalpar/deheap v0.0.0-20200318053559-9a0c2883bd56 h1:hJO00l0f92EcQn8Ygc9Y0oP++eESKvcyp+KedtfT5SQ=
+github.com/aalpar/deheap v0.0.0-20200318053559-9a0c2883bd56/go.mod h1:EJFoWbcEEVK22GYKONJjtMNamGYe6p+3x1Pr6zV5gFs=
 github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
 github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@@ -61,22 +75,20 @@ github.com/anacrolix/dms v1.1.0 h1:vbBXZS7T5FaZm+9p1pdmVVo9tN3qdc27bKSETdeT3xo=
 github.com/anacrolix/dms v1.1.0/go.mod h1:msPKAoppoNRfrYplJqx63FZ+VipDZ4Xsj3KzIQxyU7k=
 github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
 github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
-github.com/anacrolix/ffprobe v1.0.0 h1:j8fGLBsXejwdXd0pkA9iR3Dt1XwMFv5wjeYWObcue8A=
 github.com/anacrolix/ffprobe v1.0.0/go.mod h1:BIw+Bjol6CWjm/CRWrVLk2Vy+UYlkgmBZ05vpSYqZPw=
 github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/atotto/clipboard v0.1.2 h1:YZCtFu5Ie8qX2VmVTBnrqLSiU9XOWwqNRmdT3gIQzbY=
 github.com/atotto/clipboard v0.1.2/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
-github.com/aws/aws-sdk-go v1.29.9 h1:PHq9ddjfZYfCOXyqHKiCZ1CHRAk7nXhV7WTqj5l+bmQ=
-github.com/aws/aws-sdk-go v1.29.9/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/aws/aws-sdk-go v1.32.11 h1:1nYF+Tfccn/hnAZsuwPPMSCVUVnx3j6LKOpx/WhgH0A=
+github.com/aws/aws-sdk-go v1.32.11/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/billziss-gh/cgofuse v1.2.0 h1:FMdQSygSBpD4yEPENJcmvfCdmNWMVkPLlD7wWdl/7IA=
-github.com/billziss-gh/cgofuse v1.2.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
+github.com/billziss-gh/cgofuse v1.3.0 h1:mFj8XQg/vvxMFywNy1F7IqFYcMeBqceYTh1+iUhpsk8=
+github.com/billziss-gh/cgofuse v1.3.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
@@ -84,6 +96,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufo
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v1.0.1 h1:GKOz8BnRjYrb/JTKgaOk+zh26NWNdSNvdvv0xoAZMSA=
 github.com/btcsuite/btcutil v1.0.1/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
+github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
+github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
@@ -91,6 +105,9 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/calebcase/tmpfile v1.0.1 h1:vD8FSrbsbexhep39/6mvtbIHS3GzIRqiprDNCF6QqSk=
 github.com/calebcase/tmpfile v1.0.1/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
+github.com/calebcase/tmpfile v1.0.2-0.20200602150926-3af473ef8439/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
+github.com/calebcase/tmpfile v1.0.2 h1:1AGuhKiUu4J6wxz6lxuF6ck3f8G2kaV6KSEny0RGCig=
+github.com/calebcase/tmpfile v1.0.2/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -100,9 +117,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -117,13 +134,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/djherbis/times v1.2.0 h1:xANXjsC/iBqbO00vkWlYwPWgBgEVU6m6AFYg0Pic+Mc=
-github.com/djherbis/times v1.2.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible h1:DtumzkLk2zZ2SeElEr+VNz+zV7l+BTe509cV4sKPXbM=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -132,6 +150,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -141,10 +160,10 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
@@ -153,49 +172,62 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk=
-github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
 github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
-github.com/hanwen/go-fuse/v2 v2.0.3-0.20191108143333-152e6ac32d54 h1:0JL4/kY3QKTRevfl0IbEncTzA+jczGba+swfDBBluuU=
-github.com/hanwen/go-fuse/v2 v2.0.3-0.20191108143333-152e6ac32d54/go.mod h1:HH3ygZOoyRbP9y2q7y3+JM6hPL+Epe29IbWaS0UA81o=
+github.com/hanwen/go-fuse/v2 v2.0.3 h1:kpV28BKeSyVgZREItBLnaVBvOEwv2PuhNdKetwnvNHo=
+github.com/hanwen/go-fuse/v2 v2.0.3/go.mod h1:0EQM6aH2ctVpvZ6a+onrQ/vaykxh2GH7hy3e13vzTUY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -206,14 +238,15 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
-github.com/jlaffaye/ftp v0.0.0-20200422224957-b9f3ade29122 h1:dzYWuozdWNaY7mTQh5ZdmoJt2BUMavwhiux0AfGwg90=
-github.com/jlaffaye/ftp v0.0.0-20200422224957-b9f3ade29122/go.mod h1:PwUeyujmhaGohgOf0kJKxPfk3HcRv8QD/wAUN44go4k=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e h1:itZyHiOkiB8mIGouegRNLM9LttGQ3yrgRmp/J/6H/0g=
+github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -225,14 +258,14 @@ github.com/jzelinskie/whirlpool v0.0.0-20170603002051-c19460b8caa6/go.mod h1:KmH
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc h1:Y+Dob6xos1WxridFN8sIiWPOlEQzgeF9ij9dXOsXK44=
-github.com/koofr/go-httpclient v0.0.0-20190818202018-e0dc8fd921dc/go.mod h1:3xszwh+rNrYk1r9SStc4iJ326gne1OaBcrdB1ACsbzI=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 h1:Lrn8srO9JDBCf2iPjqy62stl49UDwoOxZ9/NGVi+fnk=
+github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348/go.mod h1:JBLy//Q5jzU3XSMxdONTD5EIj1LhTPktosxG2Bw1iho=
 github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a h1:02cx9xF4W2FQ1oh8CK9dWV5BnZK2mUtcbr9xR+bZiKk=
 github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
@@ -246,49 +279,51 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
-github.com/mattn/go-ieproxy v0.0.0-20200203040449-2dbc853185d9 h1:a6/tH/zhh2tdoUSDgS4gNcY6H2Mae/70R+jEkRv52ws=
-github.com/mattn/go-ieproxy v0.0.0-20200203040449-2dbc853185d9/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
+github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/minio-go/v6 v6.0.46 h1:waExJtO53xrnsNX//7cSc1h3478wqTryDx4RVD7o26I=
+github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2 h1:VlXvEx6JbFp7F9iz92zXP2Ew+9VupSpfybr+TxmjdH0=
 github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
-github.com/ncw/swift v1.0.50 h1:E01b5bVIssNhx2KnzAjMWEXkKrb8ytTqCDWY7lqmWjA=
-github.com/ncw/swift v1.0.50/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
-github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/ncw/swift v1.0.52 h1:ACF3JufDGgeKp/9mrDgQlEgS8kRYC4XKcuzj/8EJjQU=
+github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
-github.com/nsf/termbox-go v0.0.0-20200204031403-4d2b513ad8be h1:yzmWtPyxEUIKdZg4RcPq64MfS8NA6A5fNOJgYhpR9EQ=
-github.com/nsf/termbox-go v0.0.0-20200204031403-4d2b513ad8be/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
+github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1 h1:lh3PyZvY+B9nFliSGTn5uFuqQQJGuNrD0MLCokv09ag=
+github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd h1:+iAPaTbi1gZpcpDwe/BW1fx7Xoesv69hLNGPheoyhBs=
 github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
-github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
-github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -296,7 +331,6 @@ github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uC
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -305,34 +339,30 @@ github.com/pkg/sftp v1.11.0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8=
-github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
-github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 h1:w2CpS5muK+jyydnmlkqpAhzKmHmMBzBkfYUDjQNS1Dk=
-github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=
+github.com/rfjakob/eme v1.1.1 h1:t+CgvcOn+eDvj2xdglxsSnkgg8LM8jwdxnV7OnsrTn0=
+github.com/rfjakob/eme v1.1.1/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -343,16 +373,16 @@ github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
-github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
 github.com/spacemonkeygo/monkit/v3 v3.0.5/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
@@ -365,26 +395,22 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709 h1:Ko2LQMrRU+Oy/+EDBwX7eZ2jp3C47eDBB8EIhKTun+I=
 github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/t3rm1n4l/go-mega v0.0.0-20200117211730-79a813bb328d h1:GsRmok9VXIEc5B6SmRWuuO9hx4x2YBqFqgOXGt9Xs94=
-github.com/t3rm1n4l/go-mega v0.0.0-20200117211730-79a813bb328d/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 h1:IGJQmLBLYBdAknj21W3JsVof0yjEXfy1Q0K3YZebDOg=
+github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
@@ -397,8 +423,10 @@ github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/youmark/pkcs8 v0.0.0-20191102193632-94c173a94d60 h1:Ud2neINE1YFEwrcJ4EqnbRZlm9R3T8SuFKeqjIw7k44=
-github.com/youmark/pkcs8 v0.0.0-20191102193632-94c173a94d60/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce h1:F5MEHq8k6JiE10MNYaQjbKRdF1xWkOavn9aoSrHqGno=
+github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4=
 github.com/zeebo/admission/v3 v3.0.1/go.mod h1:BP3isIv9qa2A7ugEratNq1dnl2oZRXaQUGdU7WXKtbw=
@@ -409,37 +437,51 @@ github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtC
 github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo=
 github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-goftp.io/server v0.3.2 h1:bcsI4ijbvFZkA4rrUtIE/t1jNhT+0uSkiTQ4ASjZAXQ=
-goftp.io/server v0.3.2/go.mod h1:wfeAZeQgacupLVl+Ex3ozYFaAGNfCKYZiZNxLzgw6z4=
+go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM=
+go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
+goftp.io/server v0.4.0 h1:hqsVdwd1/l6QtYxD9pxca9mEAJYZ7+FPCnmeXKXHQNw=
+goftp.io/server v0.4.0/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
-golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -447,7 +489,9 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -456,18 +500,20 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -483,18 +529,25 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs=
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
@@ -503,10 +556,11 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -514,7 +568,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190415145633-3fd5a3612ccd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -524,29 +577,45 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200107144601-ef85f5a75ddf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -562,100 +631,155 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56 h1:DFtSed2q3HtNuVazwVDZ4nSRS/JrZEig0gz2BY4VNrg=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa h1:mMXQKlWCw9mIWgVLLfiycDZjMHMMYqiuakI4E/l2xcA=
+golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.21.1-0.20200411000818-c8cf5cff125e h1:apiFYpP/2xKVIDR0xuW1rrDQlSaBrCe62sQBAUthQFE=
-google.golang.org/api v0.21.1-0.20200411000818-c8cf5cff125e/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0 h1:jMF5hhVfMkTZwHW1SDpKq5CkgWLXOb31Foaca9Zr3oM=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688 h1:1+0Z5cgv1eDXJD9z2tdQF9PSSQnJXwism490hJydMRI=
-google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940 h1:MRHtG0U6SnaUb+s+LhNE1qt1FQ1wlhqr5E4usBKC0uA=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200623002339-fbb79eadd5eb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5 h1:a/Sqq5B3dGnmxhuJZIHFsIxhEkqElErr5TaU6IqBAj0=
+google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
+gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 storj.io/common v0.0.0-20200611114417-9a3d012fdb62 h1:y8vGNQ0HjtD79G8MfCwbs6hct40tSBoDaOnsxWOZpU4=
 storj.io/common v0.0.0-20200611114417-9a3d012fdb62/go.mod h1:6S6Ub92/BB+ofU7hbyPcm96b4Q1ayyN0HLog+3u+wGc=
+storj.io/common v0.0.0-20200624215549-bf610d22d466 h1:XQM2PQMJMoEaRluJOYfVONi/IclVhJgfEEPDVLAt5MU=
+storj.io/common v0.0.0-20200624215549-bf610d22d466/go.mod h1:vMAnlNbkgW6i+w/OT1h4X8w6TajOHWAT+SvFHUFCpq0=
 storj.io/drpc v0.0.12 h1:4ei1M4cnWlYxcQheX0Dg4+c12zCD+oJqfweVQVWarsA=
 storj.io/drpc v0.0.12/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA=
-storj.io/uplink v1.1.1 h1:oa5uoDtZDT58e3vy9yp24HKU1EaLs4TRM75DS+ICtqs=
-storj.io/uplink v1.1.1/go.mod h1:UkdYN/dfSgv+d8fBUoZTrX2oLdj9gzX6Q7tp3CojgKA=
+storj.io/drpc v0.0.13 h1:EDR3WiwVcIHtg+8M5vqBFmUAuJvmM2erVHIfqPPSAoc=
+storj.io/drpc v0.0.13/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA=
+storj.io/uplink v1.1.2 h1:r0EyoEDlAvqWd6SZDG10w0k2+CjSMi4wAq5J1Sw8y9Y=
+storj.io/uplink v1.1.2/go.mod h1:UkdYN/dfSgv+d8fBUoZTrX2oLdj9gzX6Q7tp3CojgKA=

lib/file/unc.go

@@ -0,0 +1,10 @@
+//+build !windows
+
+package file
+
+// UNCPath converts an absolute Windows path to a UNC long path.
+//
+// It does nothing on non windows platforms
+func UNCPath(l string) string {
+	return l
+}

lib/file/unc_test.go

@@ -0,0 +1,48 @@
+//+build windows
+
+package file
+
+import "testing"
+
+var uncTestPaths = []string{
+	`C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\server\share`,
+}
+
+var uncTestPathsResults = []string{
+	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\?\UNC\server\share`,
+}
+
+// Test that UNC paths are converted.
+func TestUncPaths(t *testing.T) {
+	for i, p := range uncTestPaths {
+		unc := UNCPath(p)
+		if unc != uncTestPathsResults[i] {
+			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
+		}
+		// Test we don't add more.
+		unc = UNCPath(unc)
+		if unc != uncTestPathsResults[i] {
+			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
+		}
+	}
+}

lib/file/unc_windows.go

@@ -0,0 +1,31 @@
+//+build windows
+
+package file
+
+import (
+	"regexp"
+	"strings"
+)
+
+// Pattern to match a windows absolute path: "c:\" and similar
+var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
+
+// UNCPath converts an absolute Windows path to a UNC long path.
+//
+// It does nothing on non windows platforms
+func UNCPath(l string) string {
+	// If prefix is "\\", we already have a UNC path or server.
+	if strings.HasPrefix(l, `\\`) {
+		// If already long path, just keep it
+		if strings.HasPrefix(l, `\\?\`) {
+			return l
+		}
+
+		// Trim "\\" from path and add UNC prefix.
+		return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
+	}
+	if isAbsWinDrive.MatchString(l) {
+		return `\\?\` + l
+	}
+	return l
+}

lib/ranges/ranges.go

@@ -0,0 +1,297 @@
+// Package ranges provides the Ranges type for keeping track of byte
+// ranges which may or may not be present in an object.
+package ranges
+
+import (
+	"sort"
+)
+
+// Range describes a single byte range
+type Range struct {
+	Pos  int64
+	Size int64
+}
+
+// End returns the end of the Range
+func (r Range) End() int64 {
+	return r.Pos + r.Size
+}
+
+// IsEmpty true if the range has no size
+func (r Range) IsEmpty() bool {
+	return r.Size <= 0
+}
+
+// Clip ensures r.End() <= offset by modifying r.Size if necessary
+//
+// if r.Pos > offset then a Range{Pos:0, Size:0} will be returned.
+func (r *Range) Clip(offset int64) {
+	if r.End() <= offset {
+		return
+	}
+	r.Size -= r.End() - offset
+	if r.Size < 0 {
+		r.Pos = 0
+		r.Size = 0
+	}
+}
+
+func min(a, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func max(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+// Intersection returns the common Range for two Range~s
+//
+// If there is no intersection then the Range returned will have
+// IsEmpty() true
+func (r Range) Intersection(b Range) (intersection Range) {
+	if (r.Pos >= b.Pos && r.Pos < b.End()) || (b.Pos >= r.Pos && b.Pos < r.End()) {
+		intersection.Pos = max(r.Pos, b.Pos)
+		intersection.Size = min(r.End(), b.End()) - intersection.Pos
+	}
+	return
+}
+
+// Ranges describes a number of Range segments. These should only be
+// added with the Ranges.Insert function. The Ranges are kept sorted
+// and coalesced to the minimum size.
+type Ranges []Range
+
+// merge the Range new into dest if possible
+//
+// dst.Pos must be >= src.Pos
+//
+// return true if merged
+func merge(new, dst *Range) bool {
+	if new.End() < dst.Pos {
+		return false
+	}
+	if new.End() > dst.End() {
+		dst.Size = new.Size
+	} else {
+		dst.Size += dst.Pos - new.Pos
+	}
+	dst.Pos = new.Pos
+	return true
+}
+
+// coalesce ranges assuming an element has been inserted at i
+func (rs *Ranges) coalesce(i int) {
+	ranges := *rs
+	var j int
+	startChop := i
+	endChop := i
+	// look at previous element too
+	if i > 0 && merge(&ranges[i-1], &ranges[i]) {
+		startChop = i - 1
+	}
+	for j = i; j < len(ranges)-1; j++ {
+		if !merge(&ranges[j], &ranges[j+1]) {
+			break
+		}
+		endChop = j + 1
+	}
+	if endChop > startChop {
+		// chop the uneeded ranges out
+		copy(ranges[startChop:], ranges[endChop:])
+		*rs = ranges[:len(ranges)-endChop+startChop]
+	}
+}
+
+// search finds the first Range in rs that has Pos >= r.Pos
+//
+// The return takes on values 0..len(rs) so may point beyond the end
+// of the slice.
+func (rs Ranges) search(r Range) int {
+	return sort.Search(len(rs), func(i int) bool {
+		return rs[i].Pos >= r.Pos
+	})
+}
+
+// Insert the new Range into a sorted and coalesced slice of
+// Ranges. The result will be sorted and coalesced.
+func (rs *Ranges) Insert(r Range) {
+	if r.IsEmpty() {
+		return
+	}
+	ranges := *rs
+	if len(ranges) == 0 {
+		ranges = append(ranges, r)
+		*rs = ranges
+		return
+	}
+	i := ranges.search(r)
+	if i == len(ranges) || !merge(&r, &ranges[i]) {
+		// insert into the range
+		ranges = append(ranges, Range{})
+		copy(ranges[i+1:], ranges[i:])
+		ranges[i] = r
+		*rs = ranges
+	}
+	rs.coalesce(i)
+}
+
+// Find searches for r in rs and returns the next present or absent
+// Range. It returns:
+//
+// curr which is the Range found
+// next is the Range which should be presented to Find next
+// present shows whether curr is present or absent
+//
+// if !next.IsEmpty() then Find should be called again with r = next
+// to retrieve the next Range.
+//
+// Note that r.Pos == curr.Pos always
+func (rs Ranges) Find(r Range) (curr, next Range, present bool) {
+	if r.IsEmpty() {
+		return r, next, false
+	}
+	var intersection Range
+	i := rs.search(r)
+	if i > 0 {
+		prev := rs[i-1]
+		// we know prev.Pos < r.Pos so intersection.Pos == r.Pos
+		intersection = prev.Intersection(r)
+		if !intersection.IsEmpty() {
+			r.Pos = intersection.End()
+			r.Size -= intersection.Size
+			return intersection, r, true
+		}
+	}
+	if i >= len(rs) {
+		return r, Range{}, false
+	}
+	found := rs[i]
+	intersection = found.Intersection(r)
+	if intersection.IsEmpty() {
+		return r, Range{}, false
+	}
+	if r.Pos < intersection.Pos {
+		curr = Range{
+			Pos:  r.Pos,
+			Size: intersection.Pos - r.Pos,
+		}
+		r.Pos = curr.End()
+		r.Size -= curr.Size
+		return curr, r, false
+	}
+	r.Pos = intersection.End()
+	r.Size -= intersection.Size
+	return intersection, r, true
+}
+
+// FoundRange is returned from FindAll
+//
+// It contains a Range and a boolean as to whether the range was
+// Present or not.
+type FoundRange struct {
+	R       Range
+	Present bool
+}
+
+// FindAll repeatedly calls Find searching for r in rs and returning
+// present or absent ranges.
+//
+// It returns a slice of FoundRange. Each element has a range and an
+// indication of whether it was present or not.
+func (rs Ranges) FindAll(r Range) (frs []FoundRange) {
+	for !r.IsEmpty() {
+		var fr FoundRange
+		fr.R, r, fr.Present = rs.Find(r)
+		frs = append(frs, fr)
+	}
+	return frs
+}
+
+// Present returns whether r can be satisfied by rs
+func (rs Ranges) Present(r Range) (present bool) {
+	if r.IsEmpty() {
+		return true
+	}
+	_, next, present := rs.Find(r)
+	if !present {
+		return false
+	}
+	if next.IsEmpty() {
+		return true
+	}
+	return false
+}
+
+// Intersection works out which ranges out of rs are entirely
+// contained within r and returns a new Ranges
+func (rs Ranges) Intersection(r Range) (newRs Ranges) {
+	if len(rs) == 0 {
+		return rs
+	}
+	for !r.IsEmpty() {
+		var curr Range
+		var found bool
+		curr, r, found = rs.Find(r)
+		if found {
+			newRs.Insert(curr)
+		}
+	}
+	return newRs
+}
+
+// Equal returns true if rs == bs
+func (rs Ranges) Equal(bs Ranges) bool {
+	if len(rs) != len(bs) {
+		return false
+	}
+	if rs == nil || bs == nil {
+		return true
+	}
+	for i := range rs {
+		if rs[i] != bs[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// Size returns the total size of all the segments
+func (rs Ranges) Size() (size int64) {
+	for _, r := range rs {
+		size += r.Size
+	}
+	return size
+}
+
+// FindMissing finds the initial part of r that is not in rs
+//
+// If r is entirely present in rs then r an empty block will be returned.
+//
+// If r is not present in rs then the block returned will have IsEmpty
+// return true.
+//
+// If r is partially present in rs then a new block will be returned
+// which starts with the first part of rs that isn't present in r. The
+// End() for this block will be the same as originally passed in.
+//
+// For all returns rout.End() == r.End()
+func (rs Ranges) FindMissing(r Range) (rout Range) {
+	rout = r
+	if r.IsEmpty() {
+		return rout
+	}
+	curr, _, present := rs.Find(r)
+	if !present {
+		// Initial block is not present
+		return rout
+	}
+	rout.Size -= curr.End() - rout.Pos
+	rout.Pos = curr.End()
+	return rout
+}

lib/ranges/ranges_test.go

@@ -0,0 +1,861 @@
+package ranges
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRangeEnd(t *testing.T) {
+	assert.Equal(t, int64(3), Range{Pos: 1, Size: 2}.End())
+}
+
+func TestRangeIsEmpty(t *testing.T) {
+	assert.Equal(t, false, Range{Pos: 1, Size: 2}.IsEmpty())
+	assert.Equal(t, true, Range{Pos: 1, Size: 0}.IsEmpty())
+	assert.Equal(t, true, Range{Pos: 1, Size: -1}.IsEmpty())
+}
+
+func TestRangeClip(t *testing.T) {
+	r := Range{Pos: 1, Size: 2}
+	r.Clip(5)
+	assert.Equal(t, Range{Pos: 1, Size: 2}, r)
+
+	r = Range{Pos: 1, Size: 6}
+	r.Clip(5)
+	assert.Equal(t, Range{Pos: 1, Size: 4}, r)
+
+	r = Range{Pos: 5, Size: 6}
+	r.Clip(5)
+	assert.Equal(t, Range{Pos: 5, Size: 0}, r)
+
+	r = Range{Pos: 7, Size: 6}
+	r.Clip(5)
+	assert.Equal(t, Range{Pos: 0, Size: 0}, r)
+}
+
+func TestRangeIntersection(t *testing.T) {
+	for _, test := range []struct {
+		r    Range
+		b    Range
+		want Range
+	}{
+		{
+			r:    Range{1, 1},
+			b:    Range{3, 1},
+			want: Range{},
+		},
+		{
+			r:    Range{1, 1},
+			b:    Range{1, 1},
+			want: Range{1, 1},
+		},
+		{
+			r:    Range{1, 9},
+			b:    Range{3, 2},
+			want: Range{3, 2},
+		},
+		{
+			r:    Range{1, 5},
+			b:    Range{3, 5},
+			want: Range{3, 3},
+		},
+	} {
+		what := fmt.Sprintf("test r=%v, b=%v", test.r, test.b)
+		got := test.r.Intersection(test.b)
+		assert.Equal(t, test.want, got, what)
+		got = test.b.Intersection(test.r)
+		assert.Equal(t, test.want, got, what)
+	}
+}
+
+func TestRangeMerge(t *testing.T) {
+	for _, test := range []struct {
+		new        Range
+		dst        Range
+		want       Range
+		wantMerged bool
+	}{
+		{
+			new:        Range{Pos: 1, Size: 1}, // .N.......
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 3, Size: 3}, // ...DDD...
+			wantMerged: false,
+		},
+		{
+			new:        Range{Pos: 1, Size: 2}, // .NN......
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 1, Size: 5}, // .XXXXX...
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 1, Size: 3}, // .NNN.....
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 1, Size: 5}, // .XXXXX...
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 1, Size: 5}, // .NNNNN...
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 1, Size: 5}, // .XXXXX...
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 1, Size: 6}, // .NNNNNN..
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 1, Size: 6}, // .XXXXXX..
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 3, Size: 3}, // ...NNN...
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 3, Size: 3}, // ...XXX...
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 3, Size: 2}, // ...NN....
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 3, Size: 3}, // ...XXX...
+			wantMerged: true,
+		},
+		{
+			new:        Range{Pos: 3, Size: 4}, // ...NNNN..
+			dst:        Range{Pos: 3, Size: 3}, // ...DDD...
+			want:       Range{Pos: 3, Size: 4}, // ...XXXX..
+			wantMerged: true,
+		},
+	} {
+		what := fmt.Sprintf("test new=%v, dst=%v", test.new, test.dst)
+		gotMerged := merge(&test.new, &test.dst)
+		assert.Equal(t, test.wantMerged, gotMerged)
+		assert.Equal(t, test.want, test.dst, what)
+	}
+}
+
+func checkRanges(t *testing.T, rs Ranges, what string) bool {
+	if len(rs) < 2 {
+		return true
+	}
+	ok := true
+	for i := 0; i < len(rs)-1; i++ {
+		a := rs[i]
+		b := rs[i+1]
+		if a.Pos >= b.Pos {
+			assert.Failf(t, "%s: Ranges in wrong order at %d in: %v", what, i, rs)
+			ok = false
+		}
+		if a.End() > b.Pos {
+			assert.Failf(t, "%s: Ranges overlap at %d in: %v", what, i, rs)
+			ok = false
+		}
+		if a.End() == b.Pos {
+			assert.Failf(t, "%s: Ranges not coalesced at %d in: %v", what, i, rs)
+			ok = false
+		}
+	}
+	return ok
+}
+
+func TestRangeCoalesce(t *testing.T) {
+	for _, test := range []struct {
+		rs   Ranges
+		i    int
+		want Ranges
+	}{
+		{
+			rs:   Ranges{},
+			want: Ranges{},
+		},
+		{
+			rs: Ranges{
+				{Pos: 1, Size: 1},
+			},
+			want: Ranges{
+				{Pos: 1, Size: 1},
+			},
+			i: 0,
+		},
+		{
+			rs: Ranges{
+				{Pos: 1, Size: 1},
+				{Pos: 2, Size: 1},
+				{Pos: 3, Size: 1},
+			},
+			want: Ranges{
+				{Pos: 1, Size: 3},
+			},
+			i: 0,
+		},
+		{
+			rs: Ranges{
+				{Pos: 1, Size: 1},
+				{Pos: 3, Size: 1},
+				{Pos: 4, Size: 1},
+				{Pos: 5, Size: 1},
+			},
+			want: Ranges{
+				{Pos: 1, Size: 1},
+				{Pos: 3, Size: 3},
+			},
+			i: 2,
+		},
+		{
+			rs:   Ranges{{38, 8}, {51, 10}, {60, 3}},
+			want: Ranges{{38, 8}, {51, 12}},
+			i:    1,
+		},
+	} {
+		got := append(Ranges{}, test.rs...)
+		got.coalesce(test.i)
+		what := fmt.Sprintf("test rs=%v, i=%d", test.rs, test.i)
+		assert.Equal(t, test.want, got, what)
+		checkRanges(t, got, what)
+	}
+}
+
+func TestRangeInsert(t *testing.T) {
+	for _, test := range []struct {
+		new  Range
+		rs   Ranges
+		want Ranges
+	}{
+		{
+			new:  Range{Pos: 1, Size: 0},
+			rs:   Ranges{},
+			want: Ranges(nil),
+		},
+		{
+			new: Range{Pos: 1, Size: 1}, // .N.......
+			rs:  Ranges{},               // .........
+			want: Ranges{ // .N.......
+				{Pos: 1, Size: 1},
+			},
+		},
+		{
+			new: Range{Pos: 1, Size: 1},    // .N.......
+			rs:  Ranges{{Pos: 5, Size: 1}}, // .....R...
+			want: Ranges{ // .N...R...
+				{Pos: 1, Size: 1},
+				{Pos: 5, Size: 1},
+			},
+		},
+		{
+			new: Range{Pos: 5, Size: 1},    // .....R...
+			rs:  Ranges{{Pos: 1, Size: 1}}, // .N.......
+			want: Ranges{ // .N...R...
+				{Pos: 1, Size: 1},
+				{Pos: 5, Size: 1},
+			},
+		},
+		{
+			new: Range{Pos: 1, Size: 1},    // .N.......
+			rs:  Ranges{{Pos: 2, Size: 1}}, // ..R......
+			want: Ranges{ // .XX......
+				{Pos: 1, Size: 2},
+			},
+		},
+		{
+			new: Range{Pos: 2, Size: 1},    // ..N.......
+			rs:  Ranges{{Pos: 1, Size: 1}}, // .R......
+			want: Ranges{ // .XX......
+				{Pos: 1, Size: 2},
+			},
+		},
+		{
+			new:  Range{Pos: 51, Size: 10},
+			rs:   Ranges{{38, 8}, {57, 2}, {60, 3}},
+			want: Ranges{{38, 8}, {51, 12}},
+		},
+	} {
+		got := append(Ranges(nil), test.rs...)
+		got.Insert(test.new)
+		what := fmt.Sprintf("test new=%v, rs=%v", test.new, test.rs)
+		assert.Equal(t, test.want, got, what)
+		checkRanges(t, test.rs, what)
+		checkRanges(t, got, what)
+	}
+}
+
+func TestRangeInsertRandom(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		var rs Ranges
+		for j := 0; j < 100; j++ {
+			var r = Range{
+				Pos:  rand.Int63n(100),
+				Size: rand.Int63n(10) + 1,
+			}
+			what := fmt.Sprintf("inserting %v into %v\n", r, rs)
+			rs.Insert(r)
+			if !checkRanges(t, rs, what) {
+				break
+			}
+			//fmt.Printf("%d: %d: %v\n", i, j, rs)
+		}
+	}
+}
+
+func TestRangeFind(t *testing.T) {
+	for _, test := range []struct {
+		rs          Ranges
+		r           Range
+		wantCurr    Range
+		wantNext    Range
+		wantPresent bool
+	}{
+		{
+			r:           Range{Pos: 1, Size: 0},
+			rs:          Ranges{},
+			wantCurr:    Range{Pos: 1, Size: 0},
+			wantNext:    Range{},
+			wantPresent: false,
+		},
+		{
+			r:           Range{Pos: 1, Size: 1},
+			rs:          Ranges{},
+			wantCurr:    Range{Pos: 1, Size: 1},
+			wantNext:    Range{},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 1, Size: 10},
+			},
+			wantCurr:    Range{Pos: 1, Size: 2},
+			wantNext:    Range{Pos: 3, Size: 0},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 1, Size: 10},
+			rs: Ranges{
+				Range{Pos: 1, Size: 2},
+			},
+			wantCurr:    Range{Pos: 1, Size: 2},
+			wantNext:    Range{Pos: 3, Size: 8},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 5, Size: 2},
+			},
+			wantCurr:    Range{Pos: 1, Size: 2},
+			wantNext:    Range{Pos: 0, Size: 0},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 2, Size: 10},
+			rs: Ranges{
+				Range{Pos: 1, Size: 2},
+			},
+			wantCurr:    Range{Pos: 2, Size: 1},
+			wantNext:    Range{Pos: 3, Size: 9},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 1, Size: 9},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantCurr:    Range{Pos: 1, Size: 1},
+			wantNext:    Range{Pos: 2, Size: 8},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 2, Size: 8},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantCurr:    Range{Pos: 2, Size: 1},
+			wantNext:    Range{Pos: 3, Size: 7},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 3, Size: 7},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantCurr:    Range{Pos: 3, Size: 1},
+			wantNext:    Range{Pos: 4, Size: 6},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 4, Size: 6},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantCurr:    Range{Pos: 4, Size: 1},
+			wantNext:    Range{Pos: 5, Size: 5},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 5, Size: 5},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantCurr:    Range{Pos: 5, Size: 5},
+			wantNext:    Range{Pos: 0, Size: 0},
+			wantPresent: false,
+		},
+	} {
+		what := fmt.Sprintf("test r=%v, rs=%v", test.r, test.rs)
+		checkRanges(t, test.rs, what)
+		gotCurr, gotNext, gotPresent := test.rs.Find(test.r)
+		assert.Equal(t, test.r.Pos, gotCurr.Pos, what)
+		assert.Equal(t, test.wantCurr, gotCurr, what)
+		assert.Equal(t, test.wantNext, gotNext, what)
+		assert.Equal(t, test.wantPresent, gotPresent, what)
+	}
+}
+
+func TestRangeFindAll(t *testing.T) {
+	for _, test := range []struct {
+		rs          Ranges
+		r           Range
+		want        []FoundRange
+		wantNext    Range
+		wantPresent bool
+	}{
+		{
+			r:    Range{Pos: 1, Size: 0},
+			rs:   Ranges{},
+			want: []FoundRange(nil),
+		},
+		{
+			r:  Range{Pos: 1, Size: 1},
+			rs: Ranges{},
+			want: []FoundRange{
+				{
+					R:       Range{Pos: 1, Size: 1},
+					Present: false,
+				},
+			},
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 1, Size: 10},
+			},
+			want: []FoundRange{
+				{
+					R:       Range{Pos: 1, Size: 2},
+					Present: true,
+				},
+			},
+		},
+		{
+			r: Range{Pos: 1, Size: 10},
+			rs: Ranges{
+				Range{Pos: 1, Size: 2},
+			},
+			want: []FoundRange{
+				{
+					R:       Range{Pos: 1, Size: 2},
+					Present: true,
+				},
+				{
+					R:       Range{Pos: 3, Size: 8},
+					Present: false,
+				},
+			},
+		},
+		{
+			r: Range{Pos: 5, Size: 5},
+			rs: Ranges{
+				Range{Pos: 4, Size: 2},
+				Range{Pos: 7, Size: 1},
+				Range{Pos: 9, Size: 2},
+			},
+			want: []FoundRange{
+				{
+					R:       Range{Pos: 5, Size: 1},
+					Present: true,
+				},
+				{
+					R:       Range{Pos: 6, Size: 1},
+					Present: false,
+				},
+				{
+					R:       Range{Pos: 7, Size: 1},
+					Present: true,
+				},
+				{
+					R:       Range{Pos: 8, Size: 1},
+					Present: false,
+				},
+				{
+					R:       Range{Pos: 9, Size: 1},
+					Present: true,
+				},
+			},
+		},
+	} {
+		what := fmt.Sprintf("test r=%v, rs=%v", test.r, test.rs)
+		checkRanges(t, test.rs, what)
+		got := test.rs.FindAll(test.r)
+		assert.Equal(t, test.want, got, what)
+	}
+}
+
+func TestRangePresent(t *testing.T) {
+	for _, test := range []struct {
+		rs          Ranges
+		r           Range
+		wantPresent bool
+	}{
+		{
+			r:           Range{Pos: 1, Size: 0},
+			rs:          Ranges{},
+			wantPresent: true,
+		},
+		{
+			r:           Range{Pos: 1, Size: 0},
+			rs:          Ranges(nil),
+			wantPresent: true,
+		},
+		{
+			r:           Range{Pos: 0, Size: 1},
+			rs:          Ranges{},
+			wantPresent: false,
+		},
+		{
+			r:           Range{Pos: 0, Size: 1},
+			rs:          Ranges(nil),
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 1, Size: 1},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 1, Size: 2},
+			},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 1, Size: 10},
+			},
+			wantPresent: true,
+		},
+		{
+			r: Range{Pos: 1, Size: 2},
+			rs: Ranges{
+				Range{Pos: 5, Size: 2},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 1, Size: 9},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 2, Size: 8},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 3, Size: 7},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 4, Size: 6},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantPresent: false,
+		},
+		{
+			r: Range{Pos: 5, Size: 5},
+			rs: Ranges{
+				Range{Pos: 2, Size: 1},
+				Range{Pos: 4, Size: 1},
+			},
+			wantPresent: false,
+		},
+	} {
+		what := fmt.Sprintf("test r=%v, rs=%v", test.r, test.rs)
+		checkRanges(t, test.rs, what)
+		gotPresent := test.rs.Present(test.r)
+		assert.Equal(t, test.wantPresent, gotPresent, what)
+		checkRanges(t, test.rs, what)
+	}
+}
+
+func TestRangesIntersection(t *testing.T) {
+	for _, test := range []struct {
+		rs   Ranges
+		r    Range
+		want Ranges
+	}{
+		{
+			rs:   Ranges(nil),
+			r:    Range{},
+			want: Ranges(nil),
+		},
+		{
+			rs:   Ranges{},
+			r:    Range{},
+			want: Ranges{},
+		},
+		{
+			rs:   Ranges{},
+			r:    Range{Pos: 1, Size: 0},
+			want: Ranges{},
+		},
+		{
+			rs:   Ranges{},
+			r:    Range{Pos: 1, Size: 1},
+			want: Ranges{},
+		},
+		{
+			rs: Ranges{{Pos: 1, Size: 5}},
+			r:  Range{Pos: 1, Size: 3},
+			want: Ranges{
+				{Pos: 1, Size: 3},
+			},
+		},
+		{
+			rs: Ranges{{Pos: 1, Size: 5}},
+			r:  Range{Pos: 1, Size: 10},
+			want: Ranges{
+				{Pos: 1, Size: 5},
+			},
+		},
+		{
+			rs: Ranges{{Pos: 1, Size: 5}},
+			r:  Range{Pos: 3, Size: 10},
+			want: Ranges{
+				{Pos: 3, Size: 3},
+			},
+		},
+		{
+			rs:   Ranges{{Pos: 1, Size: 5}},
+			r:    Range{Pos: 6, Size: 10},
+			want: Ranges(nil),
+		},
+		{
+			rs: Ranges{
+				{Pos: 1, Size: 2},
+				{Pos: 11, Size: 2},
+				{Pos: 21, Size: 2},
+				{Pos: 31, Size: 2},
+				{Pos: 41, Size: 2},
+			},
+			r: Range{Pos: 12, Size: 20},
+			want: Ranges{
+				{Pos: 12, Size: 1},
+				{Pos: 21, Size: 2},
+				{Pos: 31, Size: 1},
+			},
+		},
+	} {
+		got := test.rs.Intersection(test.r)
+		what := fmt.Sprintf("test ra=%v, r=%v", test.rs, test.r)
+		assert.Equal(t, test.want, got, what)
+		checkRanges(t, test.rs, what)
+		checkRanges(t, got, what)
+	}
+}
+
+func TestRangesEqual(t *testing.T) {
+	for _, test := range []struct {
+		rs   Ranges
+		bs   Ranges
+		want bool
+	}{
+		{
+			rs:   Ranges(nil),
+			bs:   Ranges(nil),
+			want: true,
+		},
+		{
+			rs:   Ranges{},
+			bs:   Ranges(nil),
+			want: true,
+		},
+		{
+			rs:   Ranges(nil),
+			bs:   Ranges{},
+			want: true,
+		},
+		{
+			rs:   Ranges{},
+			bs:   Ranges{},
+			want: true,
+		},
+		{
+			rs: Ranges{
+				{Pos: 0, Size: 1},
+			},
+			bs:   Ranges{},
+			want: false,
+		},
+		{
+			rs: Ranges{
+				{Pos: 0, Size: 1},
+			},
+			bs: Ranges{
+				{Pos: 0, Size: 1},
+			},
+			want: true,
+		},
+		{
+			rs: Ranges{
+				{Pos: 0, Size: 1},
+				{Pos: 10, Size: 9},
+				{Pos: 20, Size: 21},
+			},
+			bs: Ranges{
+				{Pos: 0, Size: 1},
+				{Pos: 10, Size: 9},
+				{Pos: 20, Size: 22},
+			},
+			want: false,
+		},
+		{
+			rs: Ranges{
+				{Pos: 0, Size: 1},
+				{Pos: 10, Size: 9},
+				{Pos: 20, Size: 21},
+			},
+			bs: Ranges{
+				{Pos: 0, Size: 1},
+				{Pos: 10, Size: 9},
+				{Pos: 20, Size: 21},
+			},
+			want: true,
+		},
+	} {
+		got := test.rs.Equal(test.bs)
+		what := fmt.Sprintf("test rs=%v, bs=%v", test.rs, test.bs)
+		assert.Equal(t, test.want, got, what)
+		checkRanges(t, test.bs, what)
+		checkRanges(t, test.rs, what)
+	}
+}
+
+func TestRangesSize(t *testing.T) {
+	for _, test := range []struct {
+		rs   Ranges
+		want int64
+	}{
+		{
+			rs:   Ranges(nil),
+			want: 0,
+		},
+		{
+			rs:   Ranges{},
+			want: 0,
+		},
+		{
+			rs: Ranges{
+				{Pos: 7, Size: 11},
+			},
+			want: 11,
+		},
+		{
+			rs: Ranges{
+				{Pos: 0, Size: 1},
+				{Pos: 10, Size: 9},
+				{Pos: 20, Size: 21},
+			},
+			want: 31,
+		},
+	} {
+		got := test.rs.Size()
+		what := fmt.Sprintf("test rs=%v", test.rs)
+		assert.Equal(t, test.want, got, what)
+		checkRanges(t, test.rs, what)
+	}
+}
+
+func TestFindMissing(t *testing.T) {
+	for _, test := range []struct {
+		r    Range
+		rs   Ranges
+		want Range
+	}{
+		{
+			r:    Range{},
+			rs:   Ranges(nil),
+			want: Range{},
+		},
+		{
+			r:    Range{},
+			rs:   Ranges{},
+			want: Range{},
+		},
+		{
+			r: Range{Pos: 3, Size: 5},
+			rs: Ranges{
+				{Pos: 10, Size: 5},
+				{Pos: 20, Size: 5},
+			},
+			want: Range{Pos: 3, Size: 5},
+		},
+		{
+			r: Range{Pos: 3, Size: 15},
+			rs: Ranges{
+				{Pos: 10, Size: 5},
+				{Pos: 20, Size: 5},
+			},
+			want: Range{Pos: 3, Size: 15},
+		},
+		{
+			r: Range{Pos: 10, Size: 5},
+			rs: Ranges{
+				{Pos: 10, Size: 5},
+				{Pos: 20, Size: 5},
+			},
+			want: Range{Pos: 15, Size: 0},
+		},
+		{
+			r: Range{Pos: 10, Size: 7},
+			rs: Ranges{
+				{Pos: 10, Size: 5},
+				{Pos: 20, Size: 5},
+			},
+			want: Range{Pos: 15, Size: 2},
+		},
+		{
+			r: Range{Pos: 11, Size: 7},
+			rs: Ranges{
+				{Pos: 10, Size: 5},
+				{Pos: 20, Size: 5},
+			},
+			want: Range{Pos: 15, Size: 3},
+		},
+	} {
+		got := test.rs.FindMissing(test.r)
+		what := fmt.Sprintf("test r=%v, rs=%v", test.r, test.rs)
+		assert.Equal(t, test.want, got, what)
+		assert.Equal(t, test.r.End(), got.End())
+		checkRanges(t, test.rs, what)
+	}
+}

lib/readers/pattern_reader.go

@@ -1,29 +1,59 @@
 package readers
 
-import "io"
+import (
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+// This is the smallest prime less than 256
+//
+// Using a prime here means we are less likely to hit repeating patterns
+const patternReaderModulo = 251
 
 // NewPatternReader creates a reader, that returns a deterministic byte pattern.
 // After length bytes are read
-func NewPatternReader(length int64) io.Reader {
+func NewPatternReader(length int64) io.ReadSeeker {
 	return &patternReader{
 		length: length,
 	}
 }
 
 type patternReader struct {
+	offset int64
 	length int64
 	c      byte
 }
 
 func (r *patternReader) Read(p []byte) (n int, err error) {
 	for i := range p {
-		if r.length <= 0 {
+		if r.offset >= r.length {
 			return n, io.EOF
 		}
 		p[i] = r.c
-		r.c = (r.c + 1) % 253
-		r.length--
+		r.c = (r.c + 1) % patternReaderModulo
+		r.offset++
 		n++
 	}
 	return
 }
+
+// Seek implements the io.Seeker interface.
+func (r *patternReader) Seek(offset int64, whence int) (abs int64, err error) {
+	switch whence {
+	case io.SeekStart:
+		abs = offset
+	case io.SeekCurrent:
+		abs = r.offset + offset
+	case io.SeekEnd:
+		abs = r.length + offset
+	default:
+		return 0, errors.New("patternReader: invalid whence")
+	}
+	if abs < 0 {
+		return 0, errors.New("patternReader: negative position")
+	}
+	r.offset = abs
+	r.c = byte(abs % patternReaderModulo)
+	return abs, nil
+}

lib/readers/pattern_reader_test.go

@@ -28,3 +28,61 @@ func TestPatternReader(t *testing.T) {
 	require.Equal(t, io.EOF, err)
 	require.Equal(t, 0, n)
 }
+
+func TestPatternReaderSeek(t *testing.T) {
+	r := NewPatternReader(1024)
+	b, err := ioutil.ReadAll(r)
+	require.NoError(t, err)
+
+	for i := range b {
+		assert.Equal(t, byte(i%251), b[i])
+	}
+
+	n, err := r.Seek(1, io.SeekStart)
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), n)
+
+	// pos 1
+
+	b2 := make([]byte, 10)
+	nn, err := r.Read(b2)
+	require.NoError(t, err)
+	assert.Equal(t, 10, nn)
+	assert.Equal(t, b[1:11], b2)
+
+	// pos 11
+
+	n, err = r.Seek(9, io.SeekCurrent)
+	require.NoError(t, err)
+	assert.Equal(t, int64(20), n)
+
+	// pos 20
+
+	nn, err = r.Read(b2)
+	require.NoError(t, err)
+	assert.Equal(t, 10, nn)
+	assert.Equal(t, b[20:30], b2)
+
+	n, err = r.Seek(-24, io.SeekEnd)
+	require.NoError(t, err)
+	assert.Equal(t, int64(1000), n)
+
+	// pos 1000
+
+	nn, err = r.Read(b2)
+	require.NoError(t, err)
+	assert.Equal(t, 10, nn)
+	assert.Equal(t, b[1000:1010], b2)
+
+	// Now test errors
+
+	n, err = r.Seek(1, 400)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid whence")
+	assert.Equal(t, int64(0), n)
+
+	n, err = r.Seek(-1, io.SeekStart)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "negative position")
+	assert.Equal(t, int64(0), n)
+}

vendor/bazil.org/fuse/.gitattributes

@@ -1,2 +0,0 @@
-*.go filter=gofmt
-*.cgo filter=gofmt

vendor/bazil.org/fuse/.gitignore

@@ -1,11 +0,0 @@
-*~
-.#*
-## the next line needs to start with a backslash to avoid looking like
-## a comment
-\#*#
-.*.swp
-
-*.test
-
-/clockfs
-/hellofs

vendor/bazil.org/fuse/LICENSE

@@ -1,93 +0,0 @@
-Copyright (c) 2013-2019 Tommi Virtanen.
-Copyright (c) 2009, 2011, 2012 The Go Authors.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-
-The following included software components have additional copyright
-notices and license terms that may differ from the above.
-
-
-File fuse.go:
-
-// Adapted from Plan 9 from User Space's src/cmd/9pfuse/fuse.c,
-// which carries this notice:
-//
-// The files in this directory are subject to the following license.
-//
-// The author of this software is Russ Cox.
-//
-//         Copyright (c) 2006 Russ Cox
-//
-// Permission to use, copy, modify, and distribute this software for any
-// purpose without fee is hereby granted, provided that this entire notice
-// is included in all copies of any software which is or includes a copy
-// or modification of this software and in all copies of the supporting
-// documentation for such software.
-//
-// THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED
-// WARRANTY.  IN PARTICULAR, THE AUTHOR MAKES NO REPRESENTATION OR WARRANTY
-// OF ANY KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS
-// FITNESS FOR ANY PARTICULAR PURPOSE.
-
-
-File fuse_kernel.go:
-
-// Derived from FUSE's fuse_kernel.h
-/*
-   This file defines the kernel interface of FUSE
-   Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
-
-
-   This -- and only this -- header file may also be distributed under
-   the terms of the BSD Licence as follows:
-
-   Copyright (C) 2001-2007 Miklos Szeredi. All rights reserved.
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-   1. Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-   2. Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-   ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-   ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-   OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-   HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-   LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-   OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-   SUCH DAMAGE.
-*/

vendor/bazil.org/fuse/README.md

@@ -1,23 +0,0 @@
-bazil.org/fuse -- Filesystems in Go
-===================================
-
-`bazil.org/fuse` is a Go library for writing FUSE userspace
-filesystems.
-
-It is a from-scratch implementation of the kernel-userspace
-communication protocol, and does not use the C library from the
-project called FUSE. `bazil.org/fuse` embraces Go fully for safety and
-ease of programming.
-
-Here’s how to get going:
-
-    go get bazil.org/fuse
-
-Website: http://bazil.org/fuse/
-
-Github repository: https://github.com/bazil/fuse
-
-API docs: http://godoc.org/bazil.org/fuse
-
-Our thanks to Russ Cox for his fuse library, which this project is
-based on.

vendor/bazil.org/fuse/buffer.go

@@ -1,35 +0,0 @@
-package fuse
-
-import "unsafe"
-
-// buffer provides a mechanism for constructing a message from
-// multiple segments.
-type buffer []byte
-
-// alloc allocates size bytes and returns a pointer to the new
-// segment.
-func (w *buffer) alloc(size uintptr) unsafe.Pointer {
-	s := int(size)
-	if len(*w)+s > cap(*w) {
-		old := *w
-		*w = make([]byte, len(*w), 2*cap(*w)+s)
-		copy(*w, old)
-	}
-	l := len(*w)
-	*w = (*w)[:l+s]
-	return unsafe.Pointer(&(*w)[l])
-}
-
-// reset clears out the contents of the buffer.
-func (w *buffer) reset() {
-	for i := range (*w)[:cap(*w)] {
-		(*w)[i] = 0
-	}
-	*w = (*w)[:0]
-}
-
-func newBuffer(extra uintptr) buffer {
-	const hdrSize = unsafe.Sizeof(outHeader{})
-	buf := make(buffer, hdrSize, hdrSize+extra)
-	return buf
-}

vendor/bazil.org/fuse/debug.go

@@ -1,21 +0,0 @@
-package fuse
-
-import (
-	"runtime"
-)
-
-func stack() string {
-	buf := make([]byte, 1024)
-	return string(buf[:runtime.Stack(buf, false)])
-}
-
-func nop(msg interface{}) {}
-
-// Debug is called to output debug messages, including protocol
-// traces. The default behavior is to do nothing.
-//
-// The messages have human-friendly string representations and are
-// safe to marshal to JSON.
-//
-// Implementations must not retain msg.
-var Debug func(msg interface{}) = nop

vendor/bazil.org/fuse/error_darwin.go

@@ -1,17 +0,0 @@
-package fuse
-
-import (
-	"syscall"
-)
-
-const (
-	ENOATTR = Errno(syscall.ENOATTR)
-)
-
-const (
-	errNoXattr = ENOATTR
-)
-
-func init() {
-	errnoNames[errNoXattr] = "ENOATTR"
-}

vendor/bazil.org/fuse/error_freebsd.go

@@ -1,15 +0,0 @@
-package fuse
-
-import "syscall"
-
-const (
-	ENOATTR = Errno(syscall.ENOATTR)
-)
-
-const (
-	errNoXattr = ENOATTR
-)
-
-func init() {
-	errnoNames[errNoXattr] = "ENOATTR"
-}

vendor/bazil.org/fuse/error_linux.go

@@ -1,17 +0,0 @@
-package fuse
-
-import (
-	"syscall"
-)
-
-const (
-	ENODATA = Errno(syscall.ENODATA)
-)
-
-const (
-	errNoXattr = ENODATA
-)
-
-func init() {
-	errnoNames[errNoXattr] = "ENODATA"
-}

vendor/bazil.org/fuse/error_std.go

@@ -1,31 +0,0 @@
-package fuse
-
-// There is very little commonality in extended attribute errors
-// across platforms.
-//
-// getxattr return value for "extended attribute does not exist" is
-// ENOATTR on OS X, and ENODATA on Linux and apparently at least
-// NetBSD. There may be a #define ENOATTR on Linux too, but the value
-// is ENODATA in the actual syscalls. FreeBSD and OpenBSD have no
-// ENODATA, only ENOATTR. ENOATTR is not in any of the standards,
-// ENODATA exists but is only used for STREAMs.
-//
-// Each platform will define it a errNoXattr constant, and this file
-// will enforce that it implements the right interfaces and hide the
-// implementation.
-//
-// https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man2/getxattr.2.html
-// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013090.html
-// http://mail-index.netbsd.org/tech-kern/2012/04/30/msg013097.html
-// http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html
-// http://www.freebsd.org/cgi/man.cgi?query=extattr_get_file&sektion=2
-// http://nixdoc.net/man-pages/openbsd/man2/extattr_get_file.2.html
-
-// ErrNoXattr is a platform-independent error value meaning the
-// extended attribute was not found. It can be used to respond to
-// GetxattrRequest and such.
-const ErrNoXattr = errNoXattr
-
-var _ error = ErrNoXattr
-var _ Errno = ErrNoXattr
-var _ ErrorNumber = ErrNoXattr

vendor/bazil.org/fuse/fs/tree.go

@@ -1,97 +0,0 @@
-// FUSE directory tree, for servers that wish to use it with the service loop.
-
-package fs
-
-import (
-	"context"
-	"os"
-	pathpkg "path"
-	"strings"
-	"syscall"
-
-	"bazil.org/fuse"
-)
-
-// A Tree implements a basic read-only directory tree for FUSE.
-// The Nodes contained in it may still be writable.
-type Tree struct {
-	tree
-}
-
-func (t *Tree) Root() (Node, error) {
-	return &t.tree, nil
-}
-
-// Add adds the path to the tree, resolving to the given node.
-// If path or a prefix of path has already been added to the tree,
-// Add panics.
-//
-// Add is only safe to call before starting to serve requests.
-func (t *Tree) Add(path string, node Node) {
-	path = pathpkg.Clean("/" + path)[1:]
-	elems := strings.Split(path, "/")
-	dir := Node(&t.tree)
-	for i, elem := range elems {
-		dt, ok := dir.(*tree)
-		if !ok {
-			panic("fuse: Tree.Add for " + strings.Join(elems[:i], "/") + " and " + path)
-		}
-		n := dt.lookup(elem)
-		if n != nil {
-			if i+1 == len(elems) {
-				panic("fuse: Tree.Add for " + path + " conflicts with " + elem)
-			}
-			dir = n
-		} else {
-			if i+1 == len(elems) {
-				dt.add(elem, node)
-			} else {
-				dir = &tree{}
-				dt.add(elem, dir)
-			}
-		}
-	}
-}
-
-type treeDir struct {
-	name string
-	node Node
-}
-
-type tree struct {
-	dir []treeDir
-}
-
-func (t *tree) lookup(name string) Node {
-	for _, d := range t.dir {
-		if d.name == name {
-			return d.node
-		}
-	}
-	return nil
-}
-
-func (t *tree) add(name string, n Node) {
-	t.dir = append(t.dir, treeDir{name, n})
-}
-
-func (t *tree) Attr(ctx context.Context, a *fuse.Attr) error {
-	a.Mode = os.ModeDir | 0555
-	return nil
-}
-
-func (t *tree) Lookup(ctx context.Context, name string) (Node, error) {
-	n := t.lookup(name)
-	if n != nil {
-		return n, nil
-	}
-	return nil, syscall.ENOENT
-}
-
-func (t *tree) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
-	var out []fuse.Dirent
-	for _, d := range t.dir {
-		out = append(out, fuse.Dirent{Name: d.name})
-	}
-	return out, nil
-}

vendor/bazil.org/fuse/fuse_darwin.go

@@ -1,9 +0,0 @@
-package fuse
-
-// Maximum file write size we are prepared to receive from the kernel.
-//
-// This value has to be >=16MB or OSXFUSE (3.4.0 observed) will
-// forcibly close the /dev/fuse file descriptor on a Setxattr with a
-// 16MB value. See TestSetxattr16MB and
-// https://github.com/bazil/fuse/issues/42
-const maxWrite = 16 * 1024 * 1024

vendor/bazil.org/fuse/fuse_freebsd.go

@@ -1,6 +0,0 @@
-package fuse
-
-// Maximum file write size we are prepared to receive from the kernel.
-//
-// This number is just a guess.
-const maxWrite = 128 * 1024

vendor/bazil.org/fuse/fuse_kernel.go

@@ -1,772 +0,0 @@
-// See the file LICENSE for copyright and licensing information.
-
-// Derived from FUSE's fuse_kernel.h, which carries this notice:
-/*
-   This file defines the kernel interface of FUSE
-   Copyright (C) 2001-2007  Miklos Szeredi <miklos@szeredi.hu>
-
-
-   This -- and only this -- header file may also be distributed under
-   the terms of the BSD Licence as follows:
-
-   Copyright (C) 2001-2007 Miklos Szeredi. All rights reserved.
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-   1. Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-   2. Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-   ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-   ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-   OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-   HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-   LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-   OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-   SUCH DAMAGE.
-*/
-
-package fuse
-
-import (
-	"fmt"
-	"syscall"
-	"unsafe"
-)
-
-// The FUSE version implemented by the package.
-const (
-	protoVersionMinMajor = 7
-	protoVersionMinMinor = 8
-	protoVersionMaxMajor = 7
-	protoVersionMaxMinor = 12
-)
-
-const (
-	rootID = 1
-)
-
-type kstatfs struct {
-	Blocks  uint64
-	Bfree   uint64
-	Bavail  uint64
-	Files   uint64
-	Ffree   uint64
-	Bsize   uint32
-	Namelen uint32
-	Frsize  uint32
-	_       uint32
-	Spare   [6]uint32
-}
-
-type fileLock struct {
-	Start uint64
-	End   uint64
-	Type  uint32
-	Pid   uint32
-}
-
-// GetattrFlags are bit flags that can be seen in GetattrRequest.
-type GetattrFlags uint32
-
-const (
-	// Indicates the handle is valid.
-	GetattrFh GetattrFlags = 1 << 0
-)
-
-var getattrFlagsNames = []flagName{
-	{uint32(GetattrFh), "GetattrFh"},
-}
-
-func (fl GetattrFlags) String() string {
-	return flagString(uint32(fl), getattrFlagsNames)
-}
-
-// The SetattrValid are bit flags describing which fields in the SetattrRequest
-// are included in the change.
-type SetattrValid uint32
-
-const (
-	SetattrMode   SetattrValid = 1 << 0
-	SetattrUid    SetattrValid = 1 << 1
-	SetattrGid    SetattrValid = 1 << 2
-	SetattrSize   SetattrValid = 1 << 3
-	SetattrAtime  SetattrValid = 1 << 4
-	SetattrMtime  SetattrValid = 1 << 5
-	SetattrHandle SetattrValid = 1 << 6
-
-	// Linux only(?)
-	SetattrAtimeNow  SetattrValid = 1 << 7
-	SetattrMtimeNow  SetattrValid = 1 << 8
-	SetattrLockOwner SetattrValid = 1 << 9 // http://www.mail-archive.com/git-commits-head@vger.kernel.org/msg27852.html
-
-	// OS X only
-	SetattrCrtime   SetattrValid = 1 << 28
-	SetattrChgtime  SetattrValid = 1 << 29
-	SetattrBkuptime SetattrValid = 1 << 30
-	SetattrFlags    SetattrValid = 1 << 31
-)
-
-func (fl SetattrValid) Mode() bool      { return fl&SetattrMode != 0 }
-func (fl SetattrValid) Uid() bool       { return fl&SetattrUid != 0 }
-func (fl SetattrValid) Gid() bool       { return fl&SetattrGid != 0 }
-func (fl SetattrValid) Size() bool      { return fl&SetattrSize != 0 }
-func (fl SetattrValid) Atime() bool     { return fl&SetattrAtime != 0 }
-func (fl SetattrValid) Mtime() bool     { return fl&SetattrMtime != 0 }
-func (fl SetattrValid) Handle() bool    { return fl&SetattrHandle != 0 }
-func (fl SetattrValid) AtimeNow() bool  { return fl&SetattrAtimeNow != 0 }
-func (fl SetattrValid) MtimeNow() bool  { return fl&SetattrMtimeNow != 0 }
-func (fl SetattrValid) LockOwner() bool { return fl&SetattrLockOwner != 0 }
-func (fl SetattrValid) Crtime() bool    { return fl&SetattrCrtime != 0 }
-func (fl SetattrValid) Chgtime() bool   { return fl&SetattrChgtime != 0 }
-func (fl SetattrValid) Bkuptime() bool  { return fl&SetattrBkuptime != 0 }
-func (fl SetattrValid) Flags() bool     { return fl&SetattrFlags != 0 }
-
-func (fl SetattrValid) String() string {
-	return flagString(uint32(fl), setattrValidNames)
-}
-
-var setattrValidNames = []flagName{
-	{uint32(SetattrMode), "SetattrMode"},
-	{uint32(SetattrUid), "SetattrUid"},
-	{uint32(SetattrGid), "SetattrGid"},
-	{uint32(SetattrSize), "SetattrSize"},
-	{uint32(SetattrAtime), "SetattrAtime"},
-	{uint32(SetattrMtime), "SetattrMtime"},
-	{uint32(SetattrHandle), "SetattrHandle"},
-	{uint32(SetattrAtimeNow), "SetattrAtimeNow"},
-	{uint32(SetattrMtimeNow), "SetattrMtimeNow"},
-	{uint32(SetattrLockOwner), "SetattrLockOwner"},
-	{uint32(SetattrCrtime), "SetattrCrtime"},
-	{uint32(SetattrChgtime), "SetattrChgtime"},
-	{uint32(SetattrBkuptime), "SetattrBkuptime"},
-	{uint32(SetattrFlags), "SetattrFlags"},
-}
-
-// Flags that can be seen in OpenRequest.Flags.
-const (
-	// Access modes. These are not 1-bit flags, but alternatives where
-	// only one can be chosen. See the IsReadOnly etc convenience
-	// methods.
-	OpenReadOnly  OpenFlags = syscall.O_RDONLY
-	OpenWriteOnly OpenFlags = syscall.O_WRONLY
-	OpenReadWrite OpenFlags = syscall.O_RDWR
-
-	// File was opened in append-only mode, all writes will go to end
-	// of file. OS X does not provide this information.
-	OpenAppend    OpenFlags = syscall.O_APPEND
-	OpenCreate    OpenFlags = syscall.O_CREAT
-	OpenDirectory OpenFlags = syscall.O_DIRECTORY
-	OpenExclusive OpenFlags = syscall.O_EXCL
-	OpenNonblock  OpenFlags = syscall.O_NONBLOCK
-	OpenSync      OpenFlags = syscall.O_SYNC
-	OpenTruncate  OpenFlags = syscall.O_TRUNC
-)
-
-// OpenAccessModeMask is a bitmask that separates the access mode
-// from the other flags in OpenFlags.
-const OpenAccessModeMask OpenFlags = syscall.O_ACCMODE
-
-// OpenFlags are the O_FOO flags passed to open/create/etc calls. For
-// example, os.O_WRONLY | os.O_APPEND.
-type OpenFlags uint32
-
-func (fl OpenFlags) String() string {
-	// O_RDONLY, O_RWONLY, O_RDWR are not flags
-	s := accModeName(fl & OpenAccessModeMask)
-	flags := uint32(fl &^ OpenAccessModeMask)
-	if flags != 0 {
-		s = s + "+" + flagString(flags, openFlagNames)
-	}
-	return s
-}
-
-// Return true if OpenReadOnly is set.
-func (fl OpenFlags) IsReadOnly() bool {
-	return fl&OpenAccessModeMask == OpenReadOnly
-}
-
-// Return true if OpenWriteOnly is set.
-func (fl OpenFlags) IsWriteOnly() bool {
-	return fl&OpenAccessModeMask == OpenWriteOnly
-}
-
-// Return true if OpenReadWrite is set.
-func (fl OpenFlags) IsReadWrite() bool {
-	return fl&OpenAccessModeMask == OpenReadWrite
-}
-
-func accModeName(flags OpenFlags) string {
-	switch flags {
-	case OpenReadOnly:
-		return "OpenReadOnly"
-	case OpenWriteOnly:
-		return "OpenWriteOnly"
-	case OpenReadWrite:
-		return "OpenReadWrite"
-	default:
-		return ""
-	}
-}
-
-var openFlagNames = []flagName{
-	{uint32(OpenAppend), "OpenAppend"},
-	{uint32(OpenCreate), "OpenCreate"},
-	{uint32(OpenDirectory), "OpenDirectory"},
-	{uint32(OpenExclusive), "OpenExclusive"},
-	{uint32(OpenNonblock), "OpenNonblock"},
-	{uint32(OpenSync), "OpenSync"},
-	{uint32(OpenTruncate), "OpenTruncate"},
-}
-
-// The OpenResponseFlags are returned in the OpenResponse.
-type OpenResponseFlags uint32
-
-const (
-	OpenDirectIO    OpenResponseFlags = 1 << 0 // bypass page cache for this open file
-	OpenKeepCache   OpenResponseFlags = 1 << 1 // don't invalidate the data cache on open
-	OpenNonSeekable OpenResponseFlags = 1 << 2 // mark the file as non-seekable (not supported on OS X or FreeBSD)
-
-	OpenPurgeAttr OpenResponseFlags = 1 << 30 // OS X
-	OpenPurgeUBC  OpenResponseFlags = 1 << 31 // OS X
-)
-
-func (fl OpenResponseFlags) String() string {
-	return flagString(uint32(fl), openResponseFlagNames)
-}
-
-var openResponseFlagNames = []flagName{
-	{uint32(OpenDirectIO), "OpenDirectIO"},
-	{uint32(OpenKeepCache), "OpenKeepCache"},
-	{uint32(OpenNonSeekable), "OpenNonSeekable"},
-	{uint32(OpenPurgeAttr), "OpenPurgeAttr"},
-	{uint32(OpenPurgeUBC), "OpenPurgeUBC"},
-}
-
-// The InitFlags are used in the Init exchange.
-type InitFlags uint32
-
-const (
-	InitAsyncRead     InitFlags = 1 << 0
-	InitPosixLocks    InitFlags = 1 << 1
-	InitFileOps       InitFlags = 1 << 2
-	InitAtomicTrunc   InitFlags = 1 << 3
-	InitExportSupport InitFlags = 1 << 4
-	InitBigWrites     InitFlags = 1 << 5
-	// Do not mask file access modes with umask. Not supported on OS X.
-	InitDontMask        InitFlags = 1 << 6
-	InitSpliceWrite     InitFlags = 1 << 7
-	InitSpliceMove      InitFlags = 1 << 8
-	InitSpliceRead      InitFlags = 1 << 9
-	InitFlockLocks      InitFlags = 1 << 10
-	InitHasIoctlDir     InitFlags = 1 << 11
-	InitAutoInvalData   InitFlags = 1 << 12
-	InitDoReaddirplus   InitFlags = 1 << 13
-	InitReaddirplusAuto InitFlags = 1 << 14
-	InitAsyncDIO        InitFlags = 1 << 15
-	InitWritebackCache  InitFlags = 1 << 16
-	InitNoOpenSupport   InitFlags = 1 << 17
-
-	InitCaseSensitive InitFlags = 1 << 29 // OS X only
-	InitVolRename     InitFlags = 1 << 30 // OS X only
-	InitXtimes        InitFlags = 1 << 31 // OS X only
-)
-
-type flagName struct {
-	bit  uint32
-	name string
-}
-
-var initFlagNames = []flagName{
-	{uint32(InitAsyncRead), "InitAsyncRead"},
-	{uint32(InitPosixLocks), "InitPosixLocks"},
-	{uint32(InitFileOps), "InitFileOps"},
-	{uint32(InitAtomicTrunc), "InitAtomicTrunc"},
-	{uint32(InitExportSupport), "InitExportSupport"},
-	{uint32(InitBigWrites), "InitBigWrites"},
-	{uint32(InitDontMask), "InitDontMask"},
-	{uint32(InitSpliceWrite), "InitSpliceWrite"},
-	{uint32(InitSpliceMove), "InitSpliceMove"},
-	{uint32(InitSpliceRead), "InitSpliceRead"},
-	{uint32(InitFlockLocks), "InitFlockLocks"},
-	{uint32(InitHasIoctlDir), "InitHasIoctlDir"},
-	{uint32(InitAutoInvalData), "InitAutoInvalData"},
-	{uint32(InitDoReaddirplus), "InitDoReaddirplus"},
-	{uint32(InitReaddirplusAuto), "InitReaddirplusAuto"},
-	{uint32(InitAsyncDIO), "InitAsyncDIO"},
-	{uint32(InitWritebackCache), "InitWritebackCache"},
-	{uint32(InitNoOpenSupport), "InitNoOpenSupport"},
-
-	{uint32(InitCaseSensitive), "InitCaseSensitive"},
-	{uint32(InitVolRename), "InitVolRename"},
-	{uint32(InitXtimes), "InitXtimes"},
-}
-
-func (fl InitFlags) String() string {
-	return flagString(uint32(fl), initFlagNames)
-}
-
-func flagString(f uint32, names []flagName) string {
-	var s string
-
-	if f == 0 {
-		return "0"
-	}
-
-	for _, n := range names {
-		if f&n.bit != 0 {
-			s += "+" + n.name
-			f &^= n.bit
-		}
-	}
-	if f != 0 {
-		s += fmt.Sprintf("%+#x", f)
-	}
-	return s[1:]
-}
-
-// The ReleaseFlags are used in the Release exchange.
-type ReleaseFlags uint32
-
-const (
-	ReleaseFlush ReleaseFlags = 1 << 0
-)
-
-func (fl ReleaseFlags) String() string {
-	return flagString(uint32(fl), releaseFlagNames)
-}
-
-var releaseFlagNames = []flagName{
-	{uint32(ReleaseFlush), "ReleaseFlush"},
-}
-
-// Opcodes
-const (
-	opLookup      = 1
-	opForget      = 2 // no reply
-	opGetattr     = 3
-	opSetattr     = 4
-	opReadlink    = 5
-	opSymlink     = 6
-	opMknod       = 8
-	opMkdir       = 9
-	opUnlink      = 10
-	opRmdir       = 11
-	opRename      = 12
-	opLink        = 13
-	opOpen        = 14
-	opRead        = 15
-	opWrite       = 16
-	opStatfs      = 17
-	opRelease     = 18
-	opFsync       = 20
-	opSetxattr    = 21
-	opGetxattr    = 22
-	opListxattr   = 23
-	opRemovexattr = 24
-	opFlush       = 25
-	opInit        = 26
-	opOpendir     = 27
-	opReaddir     = 28
-	opReleasedir  = 29
-	opFsyncdir    = 30
-	opGetlk       = 31
-	opSetlk       = 32
-	opSetlkw      = 33
-	opAccess      = 34
-	opCreate      = 35
-	opInterrupt   = 36
-	opBmap        = 37
-	opDestroy     = 38
-	opIoctl       = 39 // Linux?
-	opPoll        = 40 // Linux?
-
-	// OS X
-	opSetvolname = 61
-	opGetxtimes  = 62
-	opExchange   = 63
-)
-
-type entryOut struct {
-	Nodeid         uint64 // Inode ID
-	Generation     uint64 // Inode generation
-	EntryValid     uint64 // Cache timeout for the name
-	AttrValid      uint64 // Cache timeout for the attributes
-	EntryValidNsec uint32
-	AttrValidNsec  uint32
-	Attr           attr
-}
-
-func entryOutSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 9}):
-		return unsafe.Offsetof(entryOut{}.Attr) + unsafe.Offsetof(entryOut{}.Attr.Blksize)
-	default:
-		return unsafe.Sizeof(entryOut{})
-	}
-}
-
-type forgetIn struct {
-	Nlookup uint64
-}
-
-type getattrIn struct {
-	GetattrFlags uint32
-	_            uint32
-	Fh           uint64
-}
-
-type attrOut struct {
-	AttrValid     uint64 // Cache timeout for the attributes
-	AttrValidNsec uint32
-	_             uint32
-	Attr          attr
-}
-
-func attrOutSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 9}):
-		return unsafe.Offsetof(attrOut{}.Attr) + unsafe.Offsetof(attrOut{}.Attr.Blksize)
-	default:
-		return unsafe.Sizeof(attrOut{})
-	}
-}
-
-// OS X
-type getxtimesOut struct {
-	Bkuptime     uint64
-	Crtime       uint64
-	BkuptimeNsec uint32
-	CrtimeNsec   uint32
-}
-
-type mknodIn struct {
-	Mode  uint32
-	Rdev  uint32
-	Umask uint32
-	_     uint32
-	// "filename\x00" follows.
-}
-
-func mknodInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 12}):
-		return unsafe.Offsetof(mknodIn{}.Umask)
-	default:
-		return unsafe.Sizeof(mknodIn{})
-	}
-}
-
-type mkdirIn struct {
-	Mode  uint32
-	Umask uint32
-	// filename follows
-}
-
-func mkdirInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 12}):
-		return unsafe.Offsetof(mkdirIn{}.Umask) + 4
-	default:
-		return unsafe.Sizeof(mkdirIn{})
-	}
-}
-
-type renameIn struct {
-	Newdir uint64
-	// "oldname\x00newname\x00" follows
-}
-
-// OS X
-type exchangeIn struct {
-	Olddir  uint64
-	Newdir  uint64
-	Options uint64
-	// "oldname\x00newname\x00" follows
-}
-
-type linkIn struct {
-	Oldnodeid uint64
-}
-
-type setattrInCommon struct {
-	Valid     uint32
-	_         uint32
-	Fh        uint64
-	Size      uint64
-	LockOwner uint64 // unused on OS X?
-	Atime     uint64
-	Mtime     uint64
-	Unused2   uint64
-	AtimeNsec uint32
-	MtimeNsec uint32
-	Unused3   uint32
-	Mode      uint32
-	Unused4   uint32
-	Uid       uint32
-	Gid       uint32
-	Unused5   uint32
-}
-
-type openIn struct {
-	Flags  uint32
-	Unused uint32
-}
-
-type openOut struct {
-	Fh        uint64
-	OpenFlags uint32
-	_         uint32
-}
-
-type createIn struct {
-	Flags uint32
-	Mode  uint32
-	Umask uint32
-	_     uint32
-}
-
-func createInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 12}):
-		return unsafe.Offsetof(createIn{}.Umask)
-	default:
-		return unsafe.Sizeof(createIn{})
-	}
-}
-
-type releaseIn struct {
-	Fh           uint64
-	Flags        uint32
-	ReleaseFlags uint32
-	LockOwner    uint32
-}
-
-type flushIn struct {
-	Fh         uint64
-	FlushFlags uint32
-	_          uint32
-	LockOwner  uint64
-}
-
-type readIn struct {
-	Fh        uint64
-	Offset    uint64
-	Size      uint32
-	ReadFlags uint32
-	LockOwner uint64
-	Flags     uint32
-	_         uint32
-}
-
-func readInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 9}):
-		return unsafe.Offsetof(readIn{}.ReadFlags) + 4
-	default:
-		return unsafe.Sizeof(readIn{})
-	}
-}
-
-// The ReadFlags are passed in ReadRequest.
-type ReadFlags uint32
-
-const (
-	// LockOwner field is valid.
-	ReadLockOwner ReadFlags = 1 << 1
-)
-
-var readFlagNames = []flagName{
-	{uint32(ReadLockOwner), "ReadLockOwner"},
-}
-
-func (fl ReadFlags) String() string {
-	return flagString(uint32(fl), readFlagNames)
-}
-
-type writeIn struct {
-	Fh         uint64
-	Offset     uint64
-	Size       uint32
-	WriteFlags uint32
-	LockOwner  uint64
-	Flags      uint32
-	_          uint32
-}
-
-func writeInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 9}):
-		return unsafe.Offsetof(writeIn{}.LockOwner)
-	default:
-		return unsafe.Sizeof(writeIn{})
-	}
-}
-
-type writeOut struct {
-	Size uint32
-	_    uint32
-}
-
-// The WriteFlags are passed in WriteRequest.
-type WriteFlags uint32
-
-const (
-	WriteCache WriteFlags = 1 << 0
-	// LockOwner field is valid.
-	WriteLockOwner WriteFlags = 1 << 1
-)
-
-var writeFlagNames = []flagName{
-	{uint32(WriteCache), "WriteCache"},
-	{uint32(WriteLockOwner), "WriteLockOwner"},
-}
-
-func (fl WriteFlags) String() string {
-	return flagString(uint32(fl), writeFlagNames)
-}
-
-type statfsOut struct {
-	St kstatfs
-}
-
-type fsyncIn struct {
-	Fh         uint64
-	FsyncFlags uint32
-	_          uint32
-}
-
-type setxattrInCommon struct {
-	Size  uint32
-	Flags uint32
-}
-
-func (setxattrInCommon) position() uint32 {
-	return 0
-}
-
-type getxattrInCommon struct {
-	Size uint32
-	_    uint32
-}
-
-func (getxattrInCommon) position() uint32 {
-	return 0
-}
-
-type getxattrOut struct {
-	Size uint32
-	_    uint32
-}
-
-type lkIn struct {
-	Fh      uint64
-	Owner   uint64
-	Lk      fileLock
-	LkFlags uint32
-	_       uint32
-}
-
-func lkInSize(p Protocol) uintptr {
-	switch {
-	case p.LT(Protocol{7, 9}):
-		return unsafe.Offsetof(lkIn{}.LkFlags)
-	default:
-		return unsafe.Sizeof(lkIn{})
-	}
-}
-
-type lkOut struct {
-	Lk fileLock
-}
-
-type accessIn struct {
-	Mask uint32
-	_    uint32
-}
-
-type initIn struct {
-	Major        uint32
-	Minor        uint32
-	MaxReadahead uint32
-	Flags        uint32
-}
-
-const initInSize = int(unsafe.Sizeof(initIn{}))
-
-type initOut struct {
-	Major        uint32
-	Minor        uint32
-	MaxReadahead uint32
-	Flags        uint32
-	Unused       uint32
-	MaxWrite     uint32
-}
-
-type interruptIn struct {
-	Unique uint64
-}
-
-type bmapIn struct {
-	Block     uint64
-	BlockSize uint32
-	_         uint32
-}
-
-type bmapOut struct {
-	Block uint64
-}
-
-type inHeader struct {
-	Len    uint32
-	Opcode uint32
-	Unique uint64
-	Nodeid uint64
-	Uid    uint32
-	Gid    uint32
-	Pid    uint32
-	_      uint32
-}
-
-const inHeaderSize = int(unsafe.Sizeof(inHeader{}))
-
-type outHeader struct {
-	Len    uint32
-	Error  int32
-	Unique uint64
-}
-
-type dirent struct {
-	Ino     uint64
-	Off     uint64
-	Namelen uint32
-	Type    uint32
-	Name    [0]byte
-}
-
-const direntSize = 8 + 8 + 4 + 4
-
-const (
-	notifyCodePoll       int32 = 1
-	notifyCodeInvalInode int32 = 2
-	notifyCodeInvalEntry int32 = 3
-)
-
-type notifyInvalInodeOut struct {
-	Ino uint64
-	Off int64
-	Len int64
-}
-
-type notifyInvalEntryOut struct {
-	Parent  uint64
-	Namelen uint32
-	_       uint32
-}

vendor/bazil.org/fuse/fuse_kernel_darwin.go

@@ -1,88 +0,0 @@
-package fuse
-
-import (
-	"time"
-)
-
-type attr struct {
-	Ino        uint64
-	Size       uint64
-	Blocks     uint64
-	Atime      uint64
-	Mtime      uint64
-	Ctime      uint64
-	Crtime_    uint64 // OS X only
-	AtimeNsec  uint32
-	MtimeNsec  uint32
-	CtimeNsec  uint32
-	CrtimeNsec uint32 // OS X only
-	Mode       uint32
-	Nlink      uint32
-	Uid        uint32
-	Gid        uint32
-	Rdev       uint32
-	Flags_     uint32 // OS X only; see chflags(2)
-	Blksize    uint32
-	padding    uint32
-}
-
-func (a *attr) SetCrtime(s uint64, ns uint32) {
-	a.Crtime_, a.CrtimeNsec = s, ns
-}
-
-func (a *attr) SetFlags(f uint32) {
-	a.Flags_ = f
-}
-
-type setattrIn struct {
-	setattrInCommon
-
-	// OS X only
-	Bkuptime_    uint64
-	Chgtime_     uint64
-	Crtime       uint64
-	BkuptimeNsec uint32
-	ChgtimeNsec  uint32
-	CrtimeNsec   uint32
-	Flags_       uint32 // see chflags(2)
-}
-
-func (in *setattrIn) BkupTime() time.Time {
-	return time.Unix(int64(in.Bkuptime_), int64(in.BkuptimeNsec))
-}
-
-func (in *setattrIn) Chgtime() time.Time {
-	return time.Unix(int64(in.Chgtime_), int64(in.ChgtimeNsec))
-}
-
-func (in *setattrIn) Flags() uint32 {
-	return in.Flags_
-}
-
-func openFlags(flags uint32) OpenFlags {
-	return OpenFlags(flags)
-}
-
-type getxattrIn struct {
-	getxattrInCommon
-
-	// OS X only
-	Position uint32
-	Padding  uint32
-}
-
-func (g *getxattrIn) position() uint32 {
-	return g.Position
-}
-
-type setxattrIn struct {
-	setxattrInCommon
-
-	// OS X only
-	Position uint32
-	Padding  uint32
-}
-
-func (s *setxattrIn) position() uint32 {
-	return s.Position
-}

vendor/bazil.org/fuse/fuse_kernel_freebsd.go

@@ -1,62 +0,0 @@
-package fuse
-
-import "time"
-
-type attr struct {
-	Ino       uint64
-	Size      uint64
-	Blocks    uint64
-	Atime     uint64
-	Mtime     uint64
-	Ctime     uint64
-	AtimeNsec uint32
-	MtimeNsec uint32
-	CtimeNsec uint32
-	Mode      uint32
-	Nlink     uint32
-	Uid       uint32
-	Gid       uint32
-	Rdev      uint32
-	Blksize   uint32
-	padding   uint32
-}
-
-func (a *attr) Crtime() time.Time {
-	return time.Time{}
-}
-
-func (a *attr) SetCrtime(s uint64, ns uint32) {
-	// ignored on freebsd
-}
-
-func (a *attr) SetFlags(f uint32) {
-	// ignored on freebsd
-}
-
-type setattrIn struct {
-	setattrInCommon
-}
-
-func (in *setattrIn) BkupTime() time.Time {
-	return time.Time{}
-}
-
-func (in *setattrIn) Chgtime() time.Time {
-	return time.Time{}
-}
-
-func (in *setattrIn) Flags() uint32 {
-	return 0
-}
-
-func openFlags(flags uint32) OpenFlags {
-	return OpenFlags(flags)
-}
-
-type getxattrIn struct {
-	getxattrInCommon
-}
-
-type setxattrIn struct {
-	setxattrInCommon
-}

vendor/bazil.org/fuse/fuse_kernel_linux.go

@@ -1,70 +0,0 @@
-package fuse
-
-import "time"
-
-type attr struct {
-	Ino       uint64
-	Size      uint64
-	Blocks    uint64
-	Atime     uint64
-	Mtime     uint64
-	Ctime     uint64
-	AtimeNsec uint32
-	MtimeNsec uint32
-	CtimeNsec uint32
-	Mode      uint32
-	Nlink     uint32
-	Uid       uint32
-	Gid       uint32
-	Rdev      uint32
-	Blksize   uint32
-	_         uint32
-}
-
-func (a *attr) Crtime() time.Time {
-	return time.Time{}
-}
-
-func (a *attr) SetCrtime(s uint64, ns uint32) {
-	// Ignored on Linux.
-}
-
-func (a *attr) SetFlags(f uint32) {
-	// Ignored on Linux.
-}
-
-type setattrIn struct {
-	setattrInCommon
-}
-
-func (in *setattrIn) BkupTime() time.Time {
-	return time.Time{}
-}
-
-func (in *setattrIn) Chgtime() time.Time {
-	return time.Time{}
-}
-
-func (in *setattrIn) Flags() uint32 {
-	return 0
-}
-
-func openFlags(flags uint32) OpenFlags {
-	// on amd64, the 32-bit O_LARGEFILE flag is always seen;
-	// on i386, the flag probably depends on the app
-	// requesting, but in any case should be utterly
-	// uninteresting to us here; our kernel protocol messages
-	// are not directly related to the client app's kernel
-	// API/ABI
-	flags &^= 0x8000
-
-	return OpenFlags(flags)
-}
-
-type getxattrIn struct {
-	getxattrInCommon
-}
-
-type setxattrIn struct {
-	setxattrInCommon
-}

vendor/bazil.org/fuse/fuse_kernel_std.go

@@ -1 +0,0 @@
-package fuse

vendor/bazil.org/fuse/fuse_linux.go

@@ -1,7 +0,0 @@
-package fuse
-
-// Maximum file write size we are prepared to receive from the kernel.
-//
-// Linux 4.2.0 has been observed to cap this value at 128kB
-// (FUSE_MAX_PAGES_PER_REQ=32, 4kB pages).
-const maxWrite = 128 * 1024

vendor/bazil.org/fuse/fuseutil/fuseutil.go

@@ -1,20 +0,0 @@
-package fuseutil // import "bazil.org/fuse/fuseutil"
-
-import (
-	"bazil.org/fuse"
-)
-
-// HandleRead handles a read request assuming that data is the entire file content.
-// It adjusts the amount returned in resp according to req.Offset and req.Size.
-func HandleRead(req *fuse.ReadRequest, resp *fuse.ReadResponse, data []byte) {
-	if req.Offset >= int64(len(data)) {
-		data = nil
-	} else {
-		data = data[req.Offset:]
-	}
-	if len(data) > req.Size {
-		data = data[:req.Size]
-	}
-	n := copy(resp.Data[:req.Size], data)
-	resp.Data = resp.Data[:n]
-}

vendor/bazil.org/fuse/go.mod

@@ -1,8 +0,0 @@
-module bazil.org/fuse
-
-go 1.13
-
-require (
-	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
-)

vendor/bazil.org/fuse/go.sum

@@ -1,4 +0,0 @@
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/bazil.org/fuse/mount.go

@@ -1,38 +0,0 @@
-package fuse
-
-import (
-	"bufio"
-	"errors"
-	"io"
-	"log"
-	"sync"
-)
-
-var (
-	// ErrOSXFUSENotFound is returned from Mount when the OSXFUSE
-	// installation is not detected.
-	//
-	// Only happens on OS X. Make sure OSXFUSE is installed, or see
-	// OSXFUSELocations for customization.
-	ErrOSXFUSENotFound = errors.New("cannot locate OSXFUSE")
-)
-
-func neverIgnoreLine(line string) bool {
-	return false
-}
-
-func lineLogger(wg *sync.WaitGroup, prefix string, ignore func(line string) bool, r io.ReadCloser) {
-	defer wg.Done()
-
-	scanner := bufio.NewScanner(r)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if ignore(line) {
-			continue
-		}
-		log.Printf("%s: %s", prefix, line)
-	}
-	if err := scanner.Err(); err != nil {
-		log.Printf("%s, error reading: %v", prefix, err)
-	}
-}

vendor/bazil.org/fuse/mount_darwin.go

@@ -1,208 +0,0 @@
-package fuse
-
-import (
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"path"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-)
-
-var (
-	errNoAvail   = errors.New("no available fuse devices")
-	errNotLoaded = errors.New("osxfuse is not loaded")
-)
-
-func loadOSXFUSE(bin string) error {
-	cmd := exec.Command(bin)
-	cmd.Dir = "/"
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	return err
-}
-
-func openOSXFUSEDev(devPrefix string) (*os.File, error) {
-	var f *os.File
-	var err error
-	for i := uint64(0); ; i++ {
-		path := devPrefix + strconv.FormatUint(i, 10)
-		f, err = os.OpenFile(path, os.O_RDWR, 0000)
-		if os.IsNotExist(err) {
-			if i == 0 {
-				// not even the first device was found -> fuse is not loaded
-				return nil, errNotLoaded
-			}
-
-			// we've run out of kernel-provided devices
-			return nil, errNoAvail
-		}
-
-		if err2, ok := err.(*os.PathError); ok && err2.Err == syscall.EBUSY {
-			// try the next one
-			continue
-		}
-
-		if err != nil {
-			return nil, err
-		}
-		return f, nil
-	}
-}
-
-func handleMountOSXFUSE(helperName string, errCh chan<- error) func(line string) (ignore bool) {
-	var noMountpointPrefix = helperName + `: `
-	const noMountpointSuffix = `: No such file or directory`
-	return func(line string) (ignore bool) {
-		if strings.HasPrefix(line, noMountpointPrefix) && strings.HasSuffix(line, noMountpointSuffix) {
-			// re-extract it from the error message in case some layer
-			// changed the path
-			mountpoint := line[len(noMountpointPrefix) : len(line)-len(noMountpointSuffix)]
-			err := &MountpointDoesNotExistError{
-				Path: mountpoint,
-			}
-			select {
-			case errCh <- err:
-				return true
-			default:
-				// not the first error; fall back to logging it
-				return false
-			}
-		}
-
-		return false
-	}
-}
-
-// isBoringMountOSXFUSEError returns whether the Wait error is
-// uninteresting; exit status 64 is.
-func isBoringMountOSXFUSEError(err error) bool {
-	if err, ok := err.(*exec.ExitError); ok && err.Exited() {
-		if status, ok := err.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 64 {
-			return true
-		}
-	}
-	return false
-}
-
-func callMount(bin string, daemonVar string, dir string, conf *mountConfig, f *os.File, ready chan<- struct{}, errp *error) error {
-	for k, v := range conf.options {
-		if strings.Contains(k, ",") || strings.Contains(v, ",") {
-			// Silly limitation but the mount helper does not
-			// understand any escaping. See TestMountOptionCommaError.
-			return fmt.Errorf("mount options cannot contain commas on darwin: %q=%q", k, v)
-		}
-	}
-	cmd := exec.Command(
-		bin,
-		"-o", conf.getOptions(),
-		// Tell osxfuse-kext how large our buffer is. It must split
-		// writes larger than this into multiple writes.
-		//
-		// OSXFUSE seems to ignore InitResponse.MaxWrite, and uses
-		// this instead.
-		"-o", "iosize="+strconv.FormatUint(maxWrite, 10),
-		// refers to fd passed in cmd.ExtraFiles
-		"3",
-		dir,
-	)
-	cmd.ExtraFiles = []*os.File{f}
-	cmd.Env = os.Environ()
-	// OSXFUSE <3.3.0
-	cmd.Env = append(cmd.Env, "MOUNT_FUSEFS_CALL_BY_LIB=")
-	// OSXFUSE >=3.3.0
-	cmd.Env = append(cmd.Env, "MOUNT_OSXFUSE_CALL_BY_LIB=")
-
-	daemon := os.Args[0]
-	if daemonVar != "" {
-		cmd.Env = append(cmd.Env, daemonVar+"="+daemon)
-	}
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return fmt.Errorf("setting up mount_osxfusefs stderr: %v", err)
-	}
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		return fmt.Errorf("setting up mount_osxfusefs stderr: %v", err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("mount_osxfusefs: %v", err)
-	}
-	helperErrCh := make(chan error, 1)
-	go func() {
-		var wg sync.WaitGroup
-		wg.Add(2)
-		go lineLogger(&wg, "mount helper output", neverIgnoreLine, stdout)
-		helperName := path.Base(bin)
-		go lineLogger(&wg, "mount helper error", handleMountOSXFUSE(helperName, helperErrCh), stderr)
-		wg.Wait()
-		if err := cmd.Wait(); err != nil {
-			// see if we have a better error to report
-			select {
-			case helperErr := <-helperErrCh:
-				// log the Wait error if it's not what we expected
-				if !isBoringMountOSXFUSEError(err) {
-					log.Printf("mount helper failed: %v", err)
-				}
-				// and now return what we grabbed from stderr as the real
-				// error
-				*errp = helperErr
-				close(ready)
-				return
-			default:
-				// nope, fall back to generic message
-			}
-
-			*errp = fmt.Errorf("mount_osxfusefs: %v", err)
-			close(ready)
-			return
-		}
-
-		*errp = nil
-		close(ready)
-	}()
-	return nil
-}
-
-func mount(dir string, conf *mountConfig, ready chan<- struct{}, errp *error) (*os.File, error) {
-	locations := conf.osxfuseLocations
-	if locations == nil {
-		locations = []OSXFUSEPaths{
-			OSXFUSELocationV3,
-			OSXFUSELocationV2,
-		}
-	}
-	for _, loc := range locations {
-		if _, err := os.Stat(loc.Mount); os.IsNotExist(err) {
-			// try the other locations
-			continue
-		}
-
-		f, err := openOSXFUSEDev(loc.DevicePrefix)
-		if err == errNotLoaded {
-			err = loadOSXFUSE(loc.Load)
-			if err != nil {
-				return nil, err
-			}
-			// try again
-			f, err = openOSXFUSEDev(loc.DevicePrefix)
-		}
-		if err != nil {
-			return nil, err
-		}
-		err = callMount(loc.Mount, loc.DaemonVar, dir, conf, f, ready, errp)
-		if err != nil {
-			f.Close()
-			return nil, err
-		}
-		return f, nil
-	}
-	return nil, ErrOSXFUSENotFound
-}

vendor/bazil.org/fuse/mount_freebsd.go

@@ -1,111 +0,0 @@
-package fuse
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"strings"
-	"sync"
-	"syscall"
-)
-
-func handleMountFusefsStderr(errCh chan<- error) func(line string) (ignore bool) {
-	return func(line string) (ignore bool) {
-		const (
-			noMountpointPrefix = `mount_fusefs: `
-			noMountpointSuffix = `: No such file or directory`
-		)
-		if strings.HasPrefix(line, noMountpointPrefix) && strings.HasSuffix(line, noMountpointSuffix) {
-			// re-extract it from the error message in case some layer
-			// changed the path
-			mountpoint := line[len(noMountpointPrefix) : len(line)-len(noMountpointSuffix)]
-			err := &MountpointDoesNotExistError{
-				Path: mountpoint,
-			}
-			select {
-			case errCh <- err:
-				return true
-			default:
-				// not the first error; fall back to logging it
-				return false
-			}
-		}
-
-		return false
-	}
-}
-
-// isBoringMountFusefsError returns whether the Wait error is
-// uninteresting; exit status 1 is.
-func isBoringMountFusefsError(err error) bool {
-	if err, ok := err.(*exec.ExitError); ok && err.Exited() {
-		if status, ok := err.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 1 {
-			return true
-		}
-	}
-	return false
-}
-
-func mount(dir string, conf *mountConfig, ready chan<- struct{}, errp *error) (*os.File, error) {
-	for k, v := range conf.options {
-		if strings.Contains(k, ",") || strings.Contains(v, ",") {
-			// Silly limitation but the mount helper does not
-			// understand any escaping. See TestMountOptionCommaError.
-			return nil, fmt.Errorf("mount options cannot contain commas on FreeBSD: %q=%q", k, v)
-		}
-	}
-
-	f, err := os.OpenFile("/dev/fuse", os.O_RDWR, 0000)
-	if err != nil {
-		*errp = err
-		return nil, err
-	}
-
-	cmd := exec.Command(
-		"/sbin/mount_fusefs",
-		"--safe",
-		"-o", conf.getOptions(),
-		"3",
-		dir,
-	)
-	cmd.ExtraFiles = []*os.File{f}
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, fmt.Errorf("setting up mount_fusefs stderr: %v", err)
-	}
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		return nil, fmt.Errorf("setting up mount_fusefs stderr: %v", err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("mount_fusefs: %v", err)
-	}
-	helperErrCh := make(chan error, 1)
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go lineLogger(&wg, "mount helper output", neverIgnoreLine, stdout)
-	go lineLogger(&wg, "mount helper error", handleMountFusefsStderr(helperErrCh), stderr)
-	wg.Wait()
-	if err := cmd.Wait(); err != nil {
-		// see if we have a better error to report
-		select {
-		case helperErr := <-helperErrCh:
-			// log the Wait error if it's not what we expected
-			if !isBoringMountFusefsError(err) {
-				log.Printf("mount helper failed: %v", err)
-			}
-			// and now return what we grabbed from stderr as the real
-			// error
-			return nil, helperErr
-		default:
-			// nope, fall back to generic message
-		}
-		return nil, fmt.Errorf("mount_fusefs: %v", err)
-	}
-
-	close(ready)
-	return f, nil
-}

vendor/bazil.org/fuse/mount_linux.go

@@ -1,150 +0,0 @@
-package fuse
-
-import (
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"os/exec"
-	"strings"
-	"sync"
-	"syscall"
-)
-
-func handleFusermountStderr(errCh chan<- error) func(line string) (ignore bool) {
-	return func(line string) (ignore bool) {
-		if line == `fusermount: failed to open /etc/fuse.conf: Permission denied` {
-			// Silence this particular message, it occurs way too
-			// commonly and isn't very relevant to whether the mount
-			// succeeds or not.
-			return true
-		}
-
-		const (
-			noMountpointPrefix = `fusermount: failed to access mountpoint `
-			noMountpointSuffix = `: No such file or directory`
-		)
-		if strings.HasPrefix(line, noMountpointPrefix) && strings.HasSuffix(line, noMountpointSuffix) {
-			// re-extract it from the error message in case some layer
-			// changed the path
-			mountpoint := line[len(noMountpointPrefix) : len(line)-len(noMountpointSuffix)]
-			err := &MountpointDoesNotExistError{
-				Path: mountpoint,
-			}
-			select {
-			case errCh <- err:
-				return true
-			default:
-				// not the first error; fall back to logging it
-				return false
-			}
-		}
-
-		return false
-	}
-}
-
-// isBoringFusermountError returns whether the Wait error is
-// uninteresting; exit status 1 is.
-func isBoringFusermountError(err error) bool {
-	if err, ok := err.(*exec.ExitError); ok && err.Exited() {
-		if status, ok := err.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 1 {
-			return true
-		}
-	}
-	return false
-}
-
-func mount(dir string, conf *mountConfig, ready chan<- struct{}, errp *error) (fusefd *os.File, err error) {
-	// linux mount is never delayed
-	close(ready)
-
-	fds, err := syscall.Socketpair(syscall.AF_FILE, syscall.SOCK_STREAM, 0)
-	if err != nil {
-		return nil, fmt.Errorf("socketpair error: %v", err)
-	}
-
-	writeFile := os.NewFile(uintptr(fds[0]), "fusermount-child-writes")
-	defer writeFile.Close()
-
-	readFile := os.NewFile(uintptr(fds[1]), "fusermount-parent-reads")
-	defer readFile.Close()
-
-	cmd := exec.Command(
-		"fusermount",
-		"-o", conf.getOptions(),
-		"--",
-		dir,
-	)
-	cmd.Env = append(os.Environ(), "_FUSE_COMMFD=3")
-
-	cmd.ExtraFiles = []*os.File{writeFile}
-
-	var wg sync.WaitGroup
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
-	}
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("fusermount: %v", err)
-	}
-	helperErrCh := make(chan error, 1)
-	wg.Add(2)
-	go lineLogger(&wg, "mount helper output", neverIgnoreLine, stdout)
-	go lineLogger(&wg, "mount helper error", handleFusermountStderr(helperErrCh), stderr)
-	wg.Wait()
-	if err := cmd.Wait(); err != nil {
-		// see if we have a better error to report
-		select {
-		case helperErr := <-helperErrCh:
-			// log the Wait error if it's not what we expected
-			if !isBoringFusermountError(err) {
-				log.Printf("mount helper failed: %v", err)
-			}
-			// and now return what we grabbed from stderr as the real
-			// error
-			return nil, helperErr
-		default:
-			// nope, fall back to generic message
-		}
-
-		return nil, fmt.Errorf("fusermount: %v", err)
-	}
-
-	c, err := net.FileConn(readFile)
-	if err != nil {
-		return nil, fmt.Errorf("FileConn from fusermount socket: %v", err)
-	}
-	defer c.Close()
-
-	uc, ok := c.(*net.UnixConn)
-	if !ok {
-		return nil, fmt.Errorf("unexpected FileConn type; expected UnixConn, got %T", c)
-	}
-
-	buf := make([]byte, 32) // expect 1 byte
-	oob := make([]byte, 32) // expect 24 bytes
-	_, oobn, _, _, err := uc.ReadMsgUnix(buf, oob)
-	scms, err := syscall.ParseSocketControlMessage(oob[:oobn])
-	if err != nil {
-		return nil, fmt.Errorf("ParseSocketControlMessage: %v", err)
-	}
-	if len(scms) != 1 {
-		return nil, fmt.Errorf("expected 1 SocketControlMessage; got scms = %#v", scms)
-	}
-	scm := scms[0]
-	gotFds, err := syscall.ParseUnixRights(&scm)
-	if err != nil {
-		return nil, fmt.Errorf("syscall.ParseUnixRights: %v", err)
-	}
-	if len(gotFds) != 1 {
-		return nil, fmt.Errorf("wanted 1 fd; got %#v", gotFds)
-	}
-	f := os.NewFile(uintptr(gotFds[0]), "/dev/fuse")
-	return f, nil
-}

vendor/bazil.org/fuse/options.go

@@ -1,296 +0,0 @@
-package fuse
-
-import (
-	"errors"
-	"strings"
-)
-
-func dummyOption(conf *mountConfig) error {
-	return nil
-}
-
-// mountConfig holds the configuration for a mount operation.
-// Use it by passing MountOption values to Mount.
-type mountConfig struct {
-	options          map[string]string
-	maxReadahead     uint32
-	initFlags        InitFlags
-	osxfuseLocations []OSXFUSEPaths
-}
-
-func escapeComma(s string) string {
-	s = strings.Replace(s, `\`, `\\`, -1)
-	s = strings.Replace(s, `,`, `\,`, -1)
-	return s
-}
-
-// getOptions makes a string of options suitable for passing to FUSE
-// mount flag `-o`. Returns an empty string if no options were set.
-// Any platform specific adjustments should happen before the call.
-func (m *mountConfig) getOptions() string {
-	var opts []string
-	for k, v := range m.options {
-		k = escapeComma(k)
-		if v != "" {
-			k += "=" + escapeComma(v)
-		}
-		opts = append(opts, k)
-	}
-	return strings.Join(opts, ",")
-}
-
-type mountOption func(*mountConfig) error
-
-// MountOption is passed to Mount to change the behavior of the mount.
-type MountOption mountOption
-
-// FSName sets the file system name (also called source) that is
-// visible in the list of mounted file systems.
-//
-// FreeBSD ignores this option.
-func FSName(name string) MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["fsname"] = name
-		return nil
-	}
-}
-
-// Subtype sets the subtype of the mount. The main type is always
-// `fuse`. The type in a list of mounted file systems will look like
-// `fuse.foo`.
-//
-// OS X ignores this option.
-// FreeBSD ignores this option.
-func Subtype(fstype string) MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["subtype"] = fstype
-		return nil
-	}
-}
-
-// LocalVolume sets the volume to be local (instead of network),
-// changing the behavior of Finder, Spotlight, and such.
-//
-// OS X only. Others ignore this option.
-func LocalVolume() MountOption {
-	return localVolume
-}
-
-// VolumeName sets the volume name shown in Finder.
-//
-// OS X only. Others ignore this option.
-func VolumeName(name string) MountOption {
-	return volumeName(name)
-}
-
-// NoAppleDouble makes OSXFUSE disallow files with names used by OS X
-// to store extended attributes on file systems that do not support
-// them natively.
-//
-// Such file names are:
-//
-//     ._*
-//     .DS_Store
-//
-// OS X only.  Others ignore this option.
-func NoAppleDouble() MountOption {
-	return noAppleDouble
-}
-
-// NoAppleXattr makes OSXFUSE disallow extended attributes with the
-// prefix "com.apple.". This disables persistent Finder state and
-// other such information.
-//
-// OS X only.  Others ignore this option.
-func NoAppleXattr() MountOption {
-	return noAppleXattr
-}
-
-// NoBrowse makes OSXFUSE mark the volume as non-browsable, so that
-// Finder won't automatically browse it.
-//
-// OS X only.  Others ignore this option.
-func NoBrowse() MountOption {
-	return noBrowse
-}
-
-// ExclCreate causes O_EXCL flag to be set for only "truly" exclusive creates,
-// i.e. create calls for which the initiator explicitly set the O_EXCL flag.
-//
-// OSXFUSE expects all create calls to return EEXIST in case the file
-// already exists, regardless of whether O_EXCL was specified or not.
-// To ensure this behavior, it normally sets OpenExclusive for all
-// Create calls, regardless of whether the original call had it set.
-// For distributed filesystems, that may force every file create to be
-// a distributed consensus action, causing undesirable delays.
-//
-// This option makes the FUSE filesystem see the original flag value,
-// and better decide when to ensure global consensus.
-//
-// Note that returning EEXIST on existing file create is still
-// expected with OSXFUSE, regardless of the presence of the
-// OpenExclusive flag.
-//
-// For more information, see
-// https://github.com/osxfuse/osxfuse/issues/209
-//
-// OS X only. Others ignore this options.
-// Requires OSXFUSE 3.4.1 or newer.
-func ExclCreate() MountOption {
-	return exclCreate
-}
-
-// DaemonTimeout sets the time in seconds between a request and a reply before
-// the FUSE mount is declared dead.
-//
-// OS X and FreeBSD only. Others ignore this option.
-func DaemonTimeout(name string) MountOption {
-	return daemonTimeout(name)
-}
-
-// AllowOther allows other users to access the file system.
-func AllowOther() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["allow_other"] = ""
-		return nil
-	}
-}
-
-// AllowDev enables interpreting character or block special devices on the
-// filesystem.
-func AllowDev() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["dev"] = ""
-		return nil
-	}
-}
-
-// AllowSUID allows set-user-identifier or set-group-identifier bits to take
-// effect.
-func AllowSUID() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["suid"] = ""
-		return nil
-	}
-}
-
-// DefaultPermissions makes the kernel enforce access control based on
-// the file mode (as in chmod).
-//
-// Without this option, the Node itself decides what is and is not
-// allowed. This is normally ok because FUSE file systems cannot be
-// accessed by other users without AllowOther.
-//
-// FreeBSD ignores this option.
-func DefaultPermissions() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["default_permissions"] = ""
-		return nil
-	}
-}
-
-// ReadOnly makes the mount read-only.
-func ReadOnly() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["ro"] = ""
-		return nil
-	}
-}
-
-// MaxReadahead sets the number of bytes that can be prefetched for
-// sequential reads. The kernel can enforce a maximum value lower than
-// this.
-//
-// This setting makes the kernel perform speculative reads that do not
-// originate from any client process. This usually tremendously
-// improves read performance.
-func MaxReadahead(n uint32) MountOption {
-	return func(conf *mountConfig) error {
-		conf.maxReadahead = n
-		return nil
-	}
-}
-
-// AsyncRead enables multiple outstanding read requests for the same
-// handle. Without this, there is at most one request in flight at a
-// time.
-func AsyncRead() MountOption {
-	return func(conf *mountConfig) error {
-		conf.initFlags |= InitAsyncRead
-		return nil
-	}
-}
-
-// WritebackCache enables the kernel to buffer writes before sending
-// them to the FUSE server. Without this, writethrough caching is
-// used.
-func WritebackCache() MountOption {
-	return func(conf *mountConfig) error {
-		conf.initFlags |= InitWritebackCache
-		return nil
-	}
-}
-
-// OSXFUSEPaths describes the paths used by an installed OSXFUSE
-// version. See OSXFUSELocationV3 for typical values.
-type OSXFUSEPaths struct {
-	// Prefix for the device file. At mount time, an incrementing
-	// number is suffixed until a free FUSE device is found.
-	DevicePrefix string
-	// Path of the load helper, used to load the kernel extension if
-	// no device files are found.
-	Load string
-	// Path of the mount helper, used for the actual mount operation.
-	Mount string
-	// Environment variable used to pass the path to the executable
-	// calling the mount helper.
-	DaemonVar string
-}
-
-// Default paths for OSXFUSE. See OSXFUSELocations.
-var (
-	OSXFUSELocationV3 = OSXFUSEPaths{
-		DevicePrefix: "/dev/osxfuse",
-		Load:         "/Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse",
-		Mount:        "/Library/Filesystems/osxfuse.fs/Contents/Resources/mount_osxfuse",
-		DaemonVar:    "MOUNT_OSXFUSE_DAEMON_PATH",
-	}
-	OSXFUSELocationV2 = OSXFUSEPaths{
-		DevicePrefix: "/dev/osxfuse",
-		Load:         "/Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs",
-		Mount:        "/Library/Filesystems/osxfusefs.fs/Support/mount_osxfusefs",
-		DaemonVar:    "MOUNT_FUSEFS_DAEMON_PATH",
-	}
-)
-
-// OSXFUSELocations sets where to look for OSXFUSE files. The
-// arguments are all the possible locations. The previous locations
-// are replaced.
-//
-// Without this option, OSXFUSELocationV3 and OSXFUSELocationV2 are
-// used.
-//
-// OS X only. Others ignore this option.
-func OSXFUSELocations(paths ...OSXFUSEPaths) MountOption {
-	return func(conf *mountConfig) error {
-		if len(paths) == 0 {
-			return errors.New("must specify at least one location for OSXFUSELocations")
-		}
-		// replace previous values, but make a copy so there's no
-		// worries about caller mutating their slice
-		conf.osxfuseLocations = append(conf.osxfuseLocations[:0], paths...)
-		return nil
-	}
-}
-
-// AllowNonEmptyMount allows the mounting over a non-empty directory.
-//
-// The files in it will be shadowed by the freshly created mount. By
-// default these mounts are rejected to prevent accidental covering up
-// of data, which could for example prevent automatic backup.
-func AllowNonEmptyMount() MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["nonempty"] = ""
-		return nil
-	}
-}

vendor/bazil.org/fuse/options_darwin.go

@@ -1,40 +0,0 @@
-package fuse
-
-func localVolume(conf *mountConfig) error {
-	conf.options["local"] = ""
-	return nil
-}
-
-func volumeName(name string) MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["volname"] = name
-		return nil
-	}
-}
-
-func daemonTimeout(name string) MountOption {
-	return func(conf *mountConfig) error {
-		conf.options["daemon_timeout"] = name
-		return nil
-	}
-}
-
-func noAppleXattr(conf *mountConfig) error {
-	conf.options["noapplexattr"] = ""
-	return nil
-}
-
-func noAppleDouble(conf *mountConfig) error {
-	conf.options["noappledouble"] = ""
-	return nil
-}
-
-func exclCreate(conf *mountConfig) error {
-	conf.options["excl_create"] = ""
-	return nil
-}
-
-func noBrowse(conf *mountConfig) error {
-	conf.options["nobrowse"] = ""
-	return nil
-}