local: put debug statements in on Read FIXME WIP

.circleci/config.yml

@@ -1,4 +1,3 @@
----
 version: 2
 
 jobs:
@@ -42,9 +41,7 @@ jobs:
     - run:
         name: Upload artifacts
         command: |
-          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
-              make circleci_upload
-          fi
+            make circleci_upload
 
     - store_artifacts:
         path: /tmp/rclone.dist

.golangci.yml

@@ -1,26 +0,0 @@
-# golangci-lint configuration options
-
-linters:
-  enable:
-    - deadcode
-    - errcheck
-    - goimports
-    - golint
-    - ineffassign
-    - structcheck
-    - varcheck
-    - govet
-    - unconvert
-    #- prealloc
-    #- maligned
-  disable-all: true
-
-issues:
-  # Enable some lints excluded by default
-  exclude-use-default: false
-
-  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
-
-  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
-  max-same-issues: 0

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.travis.yml

@@ -1,101 +1,52 @@
----
 language: go
 sudo: required
-dist: xenial
+dist: trusty
 os:
-  - linux
+- linux
+go:
+- 1.8.x
+- 1.9.x
+- 1.10.x
+- 1.11.x
+- tip
 go_import_path: github.com/ncw/rclone
 before_install:
-  - git fetch --unshallow --tags
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-      choco install -y winfsp zip make
-      cd ../..                  # fix crlf in git checkout
-      mv $TRAVIS_REPO_SLUG _old
-      git config --global core.autocrlf false
-      git clone _old $TRAVIS_REPO_SLUG
-      cd $TRAVIS_REPO_SLUG
-    fi
+- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
+- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
 install:
-  - make vars
+- git fetch --unshallow --tags
+- make vars
+- make build_dep
+script:
+- make check
+- make quicktest
+- make compile_all
 env:
   global:
     - GOTAGS=cmount
-    - GOMAXPROCS=8              # workaround for cmd/mount tests locking up - see #3154
-    - GO111MODULE=off
-    - GITHUB_USER=ncw
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
+    - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-      - fuse
-      - libfuse-dev
-      - rpm
-      - pkg-config
+    - fuse
+    - libfuse-dev
+    - rpm
+    - pkg-config
 cache:
   directories:
     - $HOME/.cache/go-build
 matrix:
   allow_failures:
-    - go: tip
+  - go: tip
   include:
-    - go: 1.9.x
-      script:
-        - make quicktest
-    - go: 1.10.x
-      script:
-        - make quicktest
-    - go: 1.11.x
-      script:
-        - make quicktest
-    - go: 1.12.x
-      env:
-        - GOTAGS=cmount
-      script:
-        - make build_dep
-        - make check
-        - make quicktest
-        - make racequicktest
-        - make compile_all
-    - os: osx
-      go: 1.12.x
-      env:
-        - GOTAGS=  # cmount doesn't work on osx travis for some reason
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-      script:
-        - make
-        - make quicktest
-        - make racequicktest
-    # - os: windows
-    #   go: 1.12.x
-    #   env:
-    #     - GOTAGS=cmount
-    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
-    #   #filter_secrets: false     # works around a problem with secrets under windows
-    #   cache:
-    #     directories:
-    #       - ${LocalAppData}/go-build
-    #   script:
-    #     - make
-    #     - make quicktest
-    #     - make racequicktest
-    - go: tip
-      script:
-        - make quicktest
-
+  - os: osx
+    go: 1.11.x
+    env: GOTAGS=""
+    cache:
+      directories:
+        - $HOME/Library/Caches/go-build
 deploy:
   provider: script
   script: make travis_beta
@@ -103,5 +54,5 @@ deploy:
   on:
     repo: ncw/rclone
     all_branches: true
-    go: 1.12.x
-    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"
+    go: 1.11.x
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -135,7 +135,7 @@ then change into the project root and run
 
     make test
 
-This command is run daily on the integration test server. You can
+This command is run daily on the the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##

Makefile

@@ -11,12 +11,14 @@ ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	BRANCH_PATH :=
 endif
 TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 ifneq ($(TAG),$(LAST_TAG))
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+# Run full tests if go >= go1.11
+FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -24,7 +26,6 @@ BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
-LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
 .PHONY: rclone vars version
@@ -41,6 +42,7 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
+	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -55,19 +57,38 @@ test:	rclone
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-
-racequicktest:
+ifdef FULL_TESTS
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
+endif
 
 # Do source code quality checks
 check:	rclone
-	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@golangci-lint run $(LINTTAGS) ./...
-	@echo "-- END CODE QUALITY REPORT ---------------------------------"
+ifdef FULL_TESTS
+	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	errcheck $(BUILDTAGS) ./...
+	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
+	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+else
+	@echo Skipping source quality tests as version of go too old
+endif
+
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+ifdef FULL_TESTS
+	go get -u github.com/kisielk/errcheck
+	go get -u golang.org/x/tools/cmd/goimports
+	go get -u golang.org/x/lint/golint
+endif
 
 # Get the release dependencies
 release_dep:
@@ -98,7 +119,7 @@ commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
 
 backenddocs: rclone bin/make_backend_docs.py
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
+	./bin/make_backend_docs.py
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -151,7 +172,11 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
+ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+else
+	@echo Skipping compile all as version of go too old
+endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -171,15 +196,10 @@ BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
-ifeq ($(TRAVIS_OS_NAME),windows)
-# BUILD_FLAGS := -include "^windows/" -cgo
-# 386 doesn't build yet
-	BUILD_FLAGS := -include "^windows/amd64" -cgo
-endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
@@ -191,7 +211,7 @@ endif
 
 # Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -P sync $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server -v -w

README.md

@@ -7,11 +7,11 @@
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/) | 
+[G+](https://google.com/+RcloneOrg)
 
 [![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
 [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
 [![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
-[![Go Report Card](https://goreportcard.com/badge/github.com/ncw/rclone)](https://goreportcard.com/report/github.com/ncw/rclone)
 [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
 # Rclone
@@ -36,7 +36,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
-  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
@@ -45,14 +44,13 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
+  * Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
-  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -63,18 +61,16 @@ Please see [the full list of all storage providers and their features](https://r
 
 ## Features
 
-  * MD5/SHA-1 hashes checked at all times for file integrity
+  * MD5/SHA1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, e.g. two different cloud accounts
+  * Can sync to and from network, eg two different cloud accounts
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
-  * Multi-threaded downloads to local disk
-  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
 
 ## Installation & documentation
 

RELEASE.md

@@ -11,7 +11,7 @@ Making a release
   * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX.0"
+  * git commit -a -v -m "Version v1.XX"
   * make retag
   * git push --tags origin master
   * # Wait for the appveyor and travis builds to complete then...
@@ -27,7 +27,6 @@ Making a release
 
 Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
-      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files

backend/alias/alias.go

@@ -14,7 +14,7 @@ import (
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "alias",
-		Description: "Alias for an existing remote",
+		Description: "Alias for a existing remote",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "remote",
@@ -30,7 +30,7 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs constructs an Fs from the path.
+// NewFs contstructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {

backend/alias/alias_internal_test.go

@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}

backend/all/all.go

@@ -16,7 +16,6 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/jottacloud"
-	_ "github.com/ncw/rclone/backend/koofr"
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
+	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -154,7 +155,7 @@ type Fs struct {
 	noAuthClient *http.Client       // unauthenticated http client
 	root         string             // the path we are working on
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	trueRootID   string             // ID of true root directory
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
@@ -272,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:         root,
 		opt:          *opt,
 		c:            c,
-		pacer:        fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
+		pacer:        pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
 		noAuthClient: fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1092,7 +1093,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		if !bigObject {
 			in, resp, err = file.OpenHeaders(headers)
 		} else {
-			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
+			in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers)
 		}
 		return o.fs.shouldRetry(resp, err)
 	})

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 
@@ -77,7 +77,7 @@ func init() {
 		}, {
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
-			Default:  defaultUploadCutoff,
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -85,7 +85,7 @@ func init() {
 
 Note that this is stored in memory and there may be up to
 "--transfers" chunks stored at once in memory.`,
-			Default:  defaultChunkSize,
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}, {
 			Name: "list_chunk",
@@ -144,7 +144,7 @@ type Fs struct {
 	containerOKMu    sync.Mutex            // mutex to protect container OK
 	containerOK      bool                  // true if we have created the container
 	containerDeleted bool                  // true if we have deleted the container
-	pacer            *fs.Pacer             // To pace and retry the API calls
+	pacer            *pacer.Pacer          // To pace and retry the API calls
 	uploadToken      *pacer.TokenDispenser // control concurrency
 }
 
@@ -307,7 +307,7 @@ func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline
 	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -347,7 +347,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:         *opt,
 		container:   container,
 		root:        directory,
-		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetPacer(pacer.S3Pacer),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 		client:      fshttp.NewClient(fs.Config),
 	}
@@ -392,7 +392,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
 			}
 
-			f.container = parts.ContainerName
+			container = parts.ContainerName
 			containerURL = azblob.NewContainerURL(*u, pipeline)
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -754,35 +754,6 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
-// Check if the container exists
-//
-// NB this can return incorrect results if called immediately after container deletion
-func (f *Fs) dirExists() (bool, error) {
-	options := azblob.ListBlobsSegmentOptions{
-		Details: azblob.BlobListingDetails{
-			Copy:             false,
-			Metadata:         false,
-			Snapshots:        false,
-			UncommittedBlobs: false,
-			Deleted:          false,
-		},
-		MaxResults: 1,
-	}
-	err := f.pacer.Call(func() (bool, error) {
-		ctx := context.Background()
-		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
-		return f.shouldRetry(err)
-	})
-	if err == nil {
-		return true, nil
-	}
-	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
-	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
-		return false, nil
-	}
-	return false, err
-}
-
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	f.containerOKMu.Lock()
@@ -790,15 +761,6 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
-	if !f.containerDeleted {
-		exists, err := f.dirExists()
-		if err == nil {
-			f.containerOK = exists
-		}
-		if err != nil || exists {
-			return err
-		}
-	}
 
 	// now try to create the container
 	err := f.pacer.Call(func() (bool, error) {
@@ -1038,7 +1000,7 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetProper
 	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
 	o.mimeType = info.ContentType()
 	o.size = size
-	o.modTime = info.LastModified()
+	o.modTime = time.Time(info.LastModified())
 	o.accessTier = azblob.AccessTierType(info.AccessTier())
 	o.setMetadata(metadata)
 
@@ -1104,6 +1066,12 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
+// timeString returns modTime as the number of milliseconds
+// elapsed since January 1, 1970 UTC as a decimal string.
+func timeString(modTime time.Time) string {
+	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
+}
+
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1386,16 +1354,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	blob := o.getBlobReference()
 	httpHeaders := azblob.BlobHTTPHeaders{}
 	httpHeaders.ContentType = fs.MimeType(o)
-	// Compute the Content-MD5 of the file, for multiparts uploads it
-	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
-	// 		 in order to validate its integrity during transport
-	if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
-		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-		if err == nil {
-			httpHeaders.ContentMD5 = sourceMD5bytes
-		} else {
-			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+	// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
+	// MD5 only for PutBlob requests
+	if size < int64(o.fs.opt.UploadCutoff) {
+		if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				httpHeaders.ContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
 		}
 	}
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris
+// +build plan9 solaris !go1.8
 
 package azureblob

backend/b2/api/types.go

@@ -17,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal satisfies the Fatal interface
+// Fatal statisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
 	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is uninitialized
+// IsZero returns true if the timestamp is unitialised
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }
@@ -311,13 +311,3 @@ type CancelLargeFileResponse struct {
 	AccountID string `json:"accountId"` // The identifier for the account.
 	BucketID  string `json:"bucketId"`  // The unique ID of the bucket.
 }
-
-// CopyFileRequest is as passed to b2_copy_file
-type CopyFileRequest struct {
-	SourceID          string            `json:"sourceFileId"`                // The ID of the source file being copied.
-	Name              string            `json:"fileName"`                    // The name of the new file being created.
-	Range             string            `json:"range,omitempty"`             // The range of bytes to copy. If not provided, the whole source file will be copied.
-	MetadataDirective string            `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
-	ContentType       string            `json:"contentType,omitempty"`       // The MIME type of the content of the file (REPLACE only)
-	Info              map[string]string `json:"fileInfo,omitempty"`          // This field stores the metadata that will be stored with the file. (REPLACE only)
-}

backend/b2/b2.go

@@ -108,7 +108,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 Files above this size will be uploaded in chunks of "--b2-chunk-size".
 
 This value should be set no larger than 4.657GiB (== 5GB).`,
-			Default:  defaultUploadCutoff,
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -117,21 +117,8 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
 When uploading large files, chunk the file into this size.  Note that
 these chunks are buffered in memory and there might a maximum of
 "--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimum size.`,
-			Default:  defaultChunkSize,
-			Advanced: true,
-		}, {
-			Name:     "disable_checksum",
-			Help:     `Disable checksums for large (> upload cutoff) files`,
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name: "download_url",
-			Help: `Custom endpoint for downloads.
-
-This is usually set to a Cloudflare CDN URL as Backblaze offers
-free egress for data downloaded through the Cloudflare network.
-Leave blank if you want to use the endpoint provided by Backblaze.`,
+minimim size.`,
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}},
 	})
@@ -139,16 +126,14 @@ Leave blank if you want to use the endpoint provided by Backblaze.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account         string        `config:"account"`
-	Key             string        `config:"key"`
-	Endpoint        string        `config:"endpoint"`
-	TestMode        string        `config:"test_mode"`
-	Versions        bool          `config:"versions"`
-	HardDelete      bool          `config:"hard_delete"`
-	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
-	DisableCheckSum bool          `config:"disable_checksum"`
-	DownloadURL     string        `config:"download_url"`
+	Account      string        `config:"account"`
+	Key          string        `config:"key"`
+	Endpoint     string        `config:"endpoint"`
+	TestMode     string        `config:"test_mode"`
+	Versions     bool          `config:"versions"`
+	HardDelete   bool          `config:"hard_delete"`
+	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
 }
 
 // Fs represents a remote b2 server
@@ -167,7 +152,7 @@ type Fs struct {
 	uploadMu      sync.Mutex                   // lock for upload variable
 	uploads       []*api.GetUploadURLResponse  // result of get upload URL calls
 	authMu        sync.Mutex                   // lock for authorizing the account
-	pacer         *fs.Pacer                    // To pace and retry the API calls
+	pacer         *pacer.Pacer                 // To pace and retry the API calls
 	bufferTokens  chan []byte                  // control concurrency of multipart uploads
 }
 
@@ -251,7 +236,13 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 				fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err)
 			}
 		}
-		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Second)
+		retryAfterDuration := time.Duration(retryAfter) * time.Second
+		if f.pacer.GetSleep() < retryAfterDuration {
+			fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err)
+			// We set 1/2 the value here because the pacer will double it immediately
+			f.pacer.SetSleep(retryAfterDuration / 2)
+		}
+		return true, err
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -322,7 +313,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -357,7 +348,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		srv:    rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -952,13 +943,6 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
-		if apiErr, ok := err.(*api.Error); ok {
-			if apiErr.Code == "already_hidden" {
-				// sometimes eventual consistency causes this, so
-				// ignore this error since it is harmless
-				return nil
-			}
-		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1066,58 +1050,6 @@ func (f *Fs) CleanUp() error {
 	return f.purge(true)
 }
 
-// Copy src to this remote using server side copy operations.
-//
-// This is stored with the remote path given
-//
-// It returns the destination Object and a possible error
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
-	err := f.Mkdir("")
-	if err != nil {
-		return nil, err
-	}
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't copy - not same remote type")
-		return nil, fs.ErrorCantCopy
-	}
-	srcFs := srcObj.fs
-	if srcFs.bucket != f.bucket {
-		fs.Debugf(src, "Can't copy - not same bucket")
-		return nil, fs.ErrorCantCopy
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          srcObj.id,
-		Name:              f.root + remote,
-		MetadataDirective: "COPY",
-	}
-	var response api.FileInfo
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-	err = o.decodeMetaDataFileInfo(&response)
-	if err != nil {
-		return nil, err
-	}
-	return o, nil
-}
-
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.SHA1)
@@ -1206,8 +1138,17 @@ func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
 
-// getMetaData gets the metadata from the object unconditionally
-func (o *Object) getMetaData() (info *api.File, err error) {
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// Sets
+//  o.id
+//  o.modTime
+//  o.size
+//  o.sha1
+func (o *Object) readMetaData() (err error) {
+	if o.id != "" {
+		return nil
+	}
 	maxSearched := 1
 	var timestamp api.Timestamp
 	baseRemote := o.remote
@@ -1215,6 +1156,7 @@ func (o *Object) getMetaData() (info *api.File, err error) {
 		timestamp, baseRemote = api.RemoveVersion(baseRemote)
 		maxSearched = maxVersions
 	}
+	var info *api.File
 	err = o.fs.list("", true, baseRemote, maxSearched, o.fs.opt.Versions, func(remote string, object *api.File, isDirectory bool) error {
 		if isDirectory {
 			return nil
@@ -1229,30 +1171,12 @@ func (o *Object) getMetaData() (info *api.File, err error) {
 	})
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
-			return nil, fs.ErrorObjectNotFound
+			return fs.ErrorObjectNotFound
 		}
-		return nil, err
+		return err
 	}
 	if info == nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-	return info, nil
-}
-
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
-func (o *Object) readMetaData() (err error) {
-	if o.id != "" {
-		return nil
-	}
-	info, err := o.getMetaData()
-	if err != nil {
-		return err
+		return fs.ErrorObjectNotFound
 	}
 	return o.decodeMetaData(info)
 }
@@ -1273,7 +1197,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return nil
+		return err
 	}
 	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
 	return nil
@@ -1291,33 +1215,10 @@ func (o *Object) ModTime() (result time.Time) {
 	return o.modTime
 }
 
-// SetModTime sets the modification time of the Object
+// SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	info, err := o.getMetaData()
-	if err != nil {
-		return err
-	}
-	info.Info[timeKey] = timeString(modTime)
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          o.id,
-		Name:              o.fs.root + o.remote, // copy to same name
-		MetadataDirective: "REPLACE",
-		ContentType:       info.ContentType,
-		Info:              info.Info,
-	}
-	var response api.FileInfo
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, &request, &response)
-		return o.fs.shouldRetry(resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.decodeMetaDataFileInfo(&response)
+	// Not possible with B2
+	return fs.ErrorCantSetModTime
 }
 
 // Storable returns if this object is storable
@@ -1389,17 +1290,9 @@ var _ io.ReadCloser = &openFile{}
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	opts := rest.Opts{
 		Method:  "GET",
+		RootURL: o.fs.info.DownloadURL,
 		Options: options,
 	}
-
-	// Use downloadUrl from backblaze if downloadUrl is not set
-	// otherwise use the custom downloadUrl
-	if o.fs.opt.DownloadURL == "" {
-		opts.RootURL = o.fs.info.DownloadURL
-	} else {
-		opts.RootURL = o.fs.opt.DownloadURL
-	}
-
 	// Download by id if set otherwise by name
 	if o.id != "" {
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
@@ -1560,7 +1453,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// Content-Type b2/x-auto to automatically set the stored Content-Type
 	// post upload. In the case where a file extension is absent or the
 	// lookup fails, the Content-Type is set to application/octet-stream. The
-	// Content-Type mappings can be pursued here.
+	// Content-Type mappings can be purused here.
 	//
 	// X-Bz-Content-Sha1
 	// required
@@ -1607,6 +1500,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		},
 		ContentLength: &size,
 	}
+	// for go1.8 (see release notes) we must nil the Body if we want a
+	// "Content-Length: 0" header which b2 requires for all files.
+	if size == 0 {
+		opts.Body = nil
+	}
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1650,7 +1548,6 @@ func (o *Object) ID() string {
 var (
 	_ fs.Fs          = &Fs{}
 	_ fs.Purger      = &Fs{}
-	_ fs.Copier      = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.CleanUpper  = &Fs{}
 	_ fs.ListRer     = &Fs{}

backend/b2/upload.go

@@ -116,10 +116,8 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
 		},
 	}
 	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
-		}
+	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+		request.Info[sha1Key] = calculatedSha1
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {

backend/box/api/types.go

@@ -45,7 +45,7 @@ type Error struct {
 	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,7 +57,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo

backend/box/box.go

@@ -111,7 +111,7 @@ type Fs struct {
 	features     *fs.Features          // optional features
 	srv          *rest.Client          // the connection to the one drive server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *fs.Pacer             // pacer for API calls
+	pacer        *pacer.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
 }
@@ -171,13 +171,13 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
+	authRety := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
+		authRety = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // substitute reserved characters for box
@@ -260,7 +260,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:        root,
 		opt:         *opt,
 		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{
@@ -530,10 +530,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)

backend/box/upload.go

@@ -112,7 +112,7 @@ outer:
 			return shouldRetry(resp, err)
 		})
 		delay := defaultDelay
-		var why string
+		why := "unknown"
 		if err != nil {
 			// Sometimes we get 400 Error with
 			// parts_mismatch immediately after uploading
@@ -211,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		if reqSize >= int64(chunkSize) {
+			reqSize = int64(chunkSize)
 		}
 
 		// Make a block of memory

backend/cache/cache.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/ncw/rclone/backend/crypt"
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
@@ -482,7 +481,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath)
 		}
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
-		f.tempFs, err = cache.Get(f.opt.TempWritePath)
+		f.tempFs, err = fs.NewFs(f.opt.TempWritePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to create temp fs: %v", err)
 		}
@@ -577,7 +576,7 @@ The slice indices are similar to Python slices: start[:end]
 
 start is the 0 based chunk number from the beginning of the file
 to fetch inclusive. end is 0 based chunk number from the beginning
-of the file to fetch exclusive.
+of the file to fetch exclisive.
 Both values can be negative, in which case they count from the back
 of the file. The value "-5:" represents the last 5 chunks of a file.
 
@@ -871,7 +870,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 	}
 }
 
-// ChangeNotify can subscribe multiple callers
+// ChangeNotify can subsribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
@@ -1192,7 +1191,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1288,7 +1287,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1550,7 +1549,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantCopy
@@ -1626,7 +1625,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 
 	// if this is a temp object then we perform the changes locally
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantMove

backend/cache/cache_internal_test.go

@@ -387,10 +387,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), testSize)
+	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -726,7 +726,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// Call the rc function
 	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
-	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -736,21 +735,18 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
-	_, err = runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err := runInstance.list(t, rootFs, "")
 
 	// create some rand test data
 	testData2 := randStringBytes(int(chunkSize))
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
 
 	// list should have 1 item only
-	li1, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err = runInstance.list(t, rootFs, "")
 	require.Len(t, li1, 1)
 
 	// Call the rc function
 	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
-	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -758,7 +754,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// list should have 2 items now
 	li2, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
 	require.Len(t, li2, 2)
 }
 
@@ -1495,8 +1490,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 	var err error
 
 	if r.useMount {
-		var f *os.File
-		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
@@ -1506,8 +1500,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		}()
 		_, err = f.WriteString(data + append)
 	} else {
-		var obj1 fs.Object
-		obj1, err = rootFs.NewObject(src)
+		obj1, err := rootFs.NewObject(src)
 		if err != nil {
 			return err
 		}
@@ -1639,13 +1632,15 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	}
-	if f.Features().UnWrap != nil {
-		cfs, ok := f.Features().UnWrap().(*cache.Fs)
-		if ok {
-			return cfs, nil
+	} else {
+		if f.Features().UnWrap != nil {
+			cfs, ok := f.Features().UnWrap().(*cache.Fs)
+			if ok {
+				return cfs, nil
+			}
 		}
 	}
+
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/cache_test.go

@@ -15,9 +15,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   "TestCache:",
-		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		RemoteName: "TestCache:",
+		NilObject:  (*cache.Object)(nil),
 	})
 }

backend/cache/storage_persistent.go

@@ -398,7 +398,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), encoded)
+		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -809,7 +809,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), encoded)
+		err = bucket.Put([]byte(destPath), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -1023,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1049,7 +1049,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), encoded)
+			err = bucket.Put([]byte(destPath), []byte(encoded))
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}

backend/crypt/cipher.go

@@ -463,7 +463,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(newRune)
+			_, _ = result.WriteRune(rune(newRune))
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -748,7 +748,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retrieve the nonce
+	// retreive the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil

backend/crypt/crypt.go

@@ -122,7 +122,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -169,10 +169,23 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
-		SetTier:                 true,
-		GetTier:                 true,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
+	doChangeNotify := wrappedFs.Features().ChangeNotify
+	if doChangeNotify != nil {
+		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
+			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+				decrypted, err := f.DecryptFileName(path)
+				if err != nil {
+					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+					return
+				}
+				notifyFunc(decrypted, entryType)
+			}
+			doChangeNotify(wrappedNotifyFunc, pollInterval)
+		}
+	}
+
 	return f, err
 }
 
@@ -189,7 +202,6 @@ type Options struct {
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
-	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
@@ -532,16 +544,6 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
-// WrapFs returns the Fs that is wrapping this Fs
-func (f *Fs) WrapFs() fs.Fs {
-	return f.wrapper
-}
-
-// SetWrapper sets the Fs that is wrapping this Fs
-func (f *Fs) SetWrapper(wrapper fs.Fs) {
-	f.wrapper = wrapper
-}
-
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -553,7 +555,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 }
 
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calculates the hash given by HashType on the fly
+// src with it, and calcuates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
@@ -614,75 +616,6 @@ func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr
 	return m.Sums()[hashType], nil
 }
 
-// MergeDirs merges the contents of all the directories passed
-// in into the first one and rmdirs the other directories.
-func (f *Fs) MergeDirs(dirs []fs.Directory) error {
-	do := f.Fs.Features().MergeDirs
-	if do == nil {
-		return errors.New("MergeDirs not supported")
-	}
-	out := make([]fs.Directory, len(dirs))
-	for i, dir := range dirs {
-		out[i] = fs.NewDirCopy(dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
-	}
-	return do(out)
-}
-
-// DirCacheFlush resets the directory cache - used in testing
-// as an optional interface
-func (f *Fs) DirCacheFlush() {
-	do := f.Fs.Features().DirCacheFlush
-	if do != nil {
-		do()
-	}
-}
-
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(remote string) (string, error) {
-	do := f.Fs.Features().PublicLink
-	if do == nil {
-		return "", errors.New("PublicLink not supported")
-	}
-	o, err := f.NewObject(remote)
-	if err != nil {
-		// assume it is a directory
-		return do(f.cipher.EncryptDirName(remote))
-	}
-	return do(o.(*Object).Object.Remote())
-}
-
-// ChangeNotify calls the passed function with a path
-// that has had changes. If the implementation
-// uses polling, it should adhere to the given interval.
-func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
-	do := f.Fs.Features().ChangeNotify
-	if do == nil {
-		return
-	}
-	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
-		var (
-			err       error
-			decrypted string
-		)
-		switch entryType {
-		case fs.EntryDirectory:
-			decrypted, err = f.cipher.DecryptDirName(path)
-		case fs.EntryObject:
-			decrypted, err = f.cipher.DecryptFileName(path)
-		default:
-			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
-			return
-		}
-		if err != nil {
-			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-			return
-		}
-		notifyFunc(decrypted, entryType)
-	}
-	do(wrappedNotifyFunc, pollIntervalChan)
-}
-
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -841,34 +774,6 @@ func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
-// ID returns the ID of the Object if known, or "" if not
-func (o *Object) ID() string {
-	do, ok := o.Object.(fs.IDer)
-	if !ok {
-		return ""
-	}
-	return do.ID()
-}
-
-// SetTier performs changing storage tier of the Object if
-// multiple storage classes supported
-func (o *Object) SetTier(tier string) error {
-	do, ok := o.Object.(fs.SetTierer)
-	if !ok {
-		return errors.New("crypt: underlying remote does not support SetTier")
-	}
-	return do.SetTier(tier)
-}
-
-// GetTier returns storage tier or class of the Object
-func (o *Object) GetTier() string {
-	do, ok := o.Object.(fs.GetTierer)
-	if !ok {
-		return ""
-	}
-	return do.GetTier()
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -882,15 +787,7 @@ var (
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
-	_ fs.Wrapper         = (*Fs)(nil)
-	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirCacheFlusher = (*Fs)(nil)
-	_ fs.ChangeNotifier  = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_test.go

@@ -21,10 +21,8 @@ func TestIntegration(t *testing.T) {
 		t.Skip("Skipping as -remote not set")
 	}
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   *fstest.RemoteName,
-		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		RemoteName: *fstest.RemoteName,
+		NilObject:  (*crypt.Object)(nil),
 	})
 }
 
@@ -44,8 +42,6 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -65,8 +61,6 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -86,8 +80,6 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		SkipBadWindowsCharacters: true,
 	})
 }

backend/drive/drive.go

@@ -1,4 +1,7 @@
 // Package drive interfaces with the Google Drive object storage system
+
+// +build go1.9
+
 package drive
 
 // FIXME need to deal with some corner cases
@@ -18,7 +21,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -37,7 +39,6 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -53,8 +54,7 @@ const (
 	driveFolderType             = "application/vnd.google-apps.folder"
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
-	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
-	defaultBurst                = 100
+	minSleep                    = 10 * time.Millisecond
 	defaultExportExtensions     = "docx,xlsx,pptx,svg"
 	scopePrefix                 = "https://www.googleapis.com/auth/"
 	defaultScope                = "drive"
@@ -183,10 +183,10 @@ func init() {
 		},
 		Options: []fs.Option{{
 			Name: config.ConfigClientID,
-			Help: "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance.",
+			Help: "Google Application Client Id\nLeave blank normally.",
 		}, {
 			Name: config.ConfigClientSecret,
-			Help: "Google Application Client Secret\nSetting your own is recommended.",
+			Help: "Google Application Client Secret\nLeave blank normally.",
 		}, {
 			Name: "scope",
 			Help: "Scope that rclone should use when requesting access from drive.",
@@ -237,22 +237,6 @@ func init() {
 			Default:  false,
 			Help:     "Skip google documents in all listings.\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
-		}, {
-			Name:    "skip_checksum_gphotos",
-			Default: false,
-			Help: `Skip MD5 checksum on Google photos and videos only.
-
-Use this if you get checksum errors when transferring Google photos or
-videos.
-
-Setting this flag will cause Google photos and videos to return a
-blank MD5 checksum.
-
-Google photos are identifed by being in the "photos" space.
-
-Corrupted checksums are caused by Google modifying the image/video but
-not updating the checksum.`,
-			Advanced: true,
 		}, {
 			Name:    "shared_with_me",
 			Default: false,
@@ -367,39 +351,11 @@ will download it anyway.`,
 			Default:  false,
 			Help:     "Keep new head revision of each file forever.",
 			Advanced: true,
-		}, {
-			Name:    "size_as_quota",
-			Default: false,
-			Help: `Show storage quota usage for file size.
-
-The storage used by a file is the size of the current version plus any
-older versions that have been set to keep forever.`,
-			Advanced: true,
 		}, {
 			Name:     "v2_download_min_size",
 			Default:  fs.SizeSuffix(-1),
 			Help:     "If Object's are greater, use drive v2 API to download.",
 			Advanced: true,
-		}, {
-			Name:     "pacer_min_sleep",
-			Default:  defaultMinSleep,
-			Help:     "Minimum time to sleep between API calls.",
-			Advanced: true,
-		}, {
-			Name:     "pacer_burst",
-			Default:  defaultBurst,
-			Help:     "Number of API calls to allow without sleeping.",
-			Advanced: true,
-		}, {
-			Name:    "server_side_across_configs",
-			Default: false,
-			Help: `Allow server side operations (eg copy) to work across different drive configs.
-
-This can be useful if you wish to do a server side copy between two
-different Google drives.  Note that this isn't enabled by default
-because it isn't easy to tell if it will work beween any two
-configurations.`,
-			Advanced: true,
 		}},
 	})
 
@@ -427,7 +383,6 @@ type Options struct {
 	AuthOwnerOnly             bool          `config:"auth_owner_only"`
 	UseTrash                  bool          `config:"use_trash"`
 	SkipGdocs                 bool          `config:"skip_gdocs"`
-	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool          `config:"shared_with_me"`
 	TrashedOnly               bool          `config:"trashed_only"`
 	Extensions                string        `config:"formats"`
@@ -442,11 +397,7 @@ type Options struct {
 	ChunkSize                 fs.SizeSuffix `config:"chunk_size"`
 	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
 	KeepRevisionForever       bool          `config:"keep_revision_forever"`
-	SizeAsQuota               bool          `config:"size_as_quota"`
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
-	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
-	PacerBurst                int           `config:"pacer_burst"`
-	ServerSideAcrossConfigs   bool          `config:"server_side_across_configs"`
 }
 
 // Fs represents a remote drive server
@@ -460,7 +411,7 @@ type Fs struct {
 	client           *http.Client       // authorized client
 	rootFolderID     string             // the id of the root folder
 	dirCache         *dircache.DirCache // Map of directory path to directory id
-	pacer            *fs.Pacer          // To pace the API calls
+	pacer            *pacer.Pacer       // To pace the API calls
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
@@ -516,7 +467,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whether a given err rates being retried
+// shouldRetry determines whehter a given err rates being retried
 func shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
@@ -649,12 +600,6 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, inc
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
-	if f.opt.SkipChecksumGphotos {
-		fields += ",spaces"
-	}
-	if f.opt.SizeAsQuota {
-		fields += ",quotaBytesUsed"
-	}
 
 	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
 
@@ -716,33 +661,28 @@ func isPowerOfTwo(x int64) bool {
 }
 
 // add a charset parameter to all text/* MIME types
-func fixMimeType(mimeTypeIn string) string {
-	if mimeTypeIn == "" {
-		return ""
-	}
-	mediaType, param, err := mime.ParseMediaType(mimeTypeIn)
+func fixMimeType(mimeType string) string {
+	mediaType, param, err := mime.ParseMediaType(mimeType)
 	if err != nil {
-		return mimeTypeIn
+		return mimeType
 	}
-	mimeTypeOut := mimeTypeIn
-	if strings.HasPrefix(mediaType, "text/") && param["charset"] == "" {
+	if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
 		param["charset"] = "utf-8"
-		mimeTypeOut = mime.FormatMediaType(mediaType, param)
+		mimeType = mime.FormatMediaType(mediaType, param)
 	}
-	if mimeTypeOut == "" {
-		panic(errors.Errorf("unable to fix MIME type %q", mimeTypeIn))
-	}
-	return mimeTypeOut
+	return mimeType
 }
-func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
-	out = make(map[string][]string, len(in))
-	for k, v := range in {
+func fixMimeTypeMap(m map[string][]string) map[string][]string {
+	for _, v := range m {
 		for i, mt := range v {
-			v[i] = fixMimeType(mt)
+			fixed := fixMimeType(mt)
+			if fixed == "" {
+				panic(errors.Errorf("unable to fix MIME type %q", mt))
+			}
+			v[i] = fixed
 		}
-		out[fixMimeType(k)] = v
 	}
-	return out
+	return m
 }
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
@@ -804,7 +744,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 	listFailed := false
 	for {
 		var teamDrives *drive.TeamDriveList
-		err = newPacer(opt).Call(func() (bool, error) {
+		err = newPacer().Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Do()
 			return shouldRetry(err)
 		})
@@ -834,8 +774,8 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer(opt *Options) *fs.Pacer {
-	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
+func newPacer() *pacer.Pacer {
+	return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
@@ -908,7 +848,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -939,7 +879,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		pacer: newPacer(opt),
+		pacer: newPacer(),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
@@ -947,7 +887,6 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
-		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f)
 
 	// Create a new authorized Drive client.
@@ -1030,31 +969,18 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 	if f.opt.UseCreatedDate {
 		modifiedDate = info.CreatedTime
 	}
-	size := info.Size
-	if f.opt.SizeAsQuota {
-		size = info.QuotaBytesUsed
-	}
 	return baseObject{
 		fs:           f,
 		remote:       remote,
 		id:           info.Id,
 		modifiedDate: modifiedDate,
 		mimeType:     info.MimeType,
-		bytes:        size,
+		bytes:        info.Size,
 	}
 }
 
 // newRegularObject creates a fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
-	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
-	if f.opt.SkipChecksumGphotos {
-		for _, space := range info.Spaces {
-			if space == "photos" {
-				info.Md5Checksum = ""
-				break
-			}
-		}
-	}
 	return &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, info.Id),
@@ -1399,46 +1325,17 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-// listREntry is a task to be executed by a litRRunner
-type listREntry struct {
-	id, path string
-}
-
-// listRSlices is a helper struct to sort two slices at once
-type listRSlices struct {
-	dirs  []string
-	paths []string
-}
-
-func (s listRSlices) Sort() {
-	sort.Sort(s)
-}
-
-func (s listRSlices) Len() int {
-	return len(s.dirs)
-}
-
-func (s listRSlices) Swap(i, j int) {
-	s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
-	s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
-}
-
-func (s listRSlices) Less(i, j int) bool {
-	return s.dirs[i] < s.dirs[j]
-}
-
 // listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
 //
-// In each cycle it will read up to grouping entries from the in channel without blocking.
+// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
+func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
 	var dirs []string
-	var paths []string
 
 	for dir := range in {
-		dirs = append(dirs[:0], dir.id)
-		paths = append(paths[:0], dir.path)
+		dirs = append(dirs[:0], dir)
+		wait := time.After(10 * time.Millisecond)
 	waitloop:
 		for i := 1; i < grouping; i++ {
 			select {
@@ -1446,32 +1343,31 @@ func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- er
 				if !ok {
 					break waitloop
 				}
-				dirs = append(dirs, d.id)
-				paths = append(paths, d.path)
-			default:
+				dirs = append(dirs, d)
+			case <-wait:
+				break waitloop
 			}
 		}
-		listRSlices{dirs, paths}.Sort()
 		var iErr error
 		_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
-			for _, parent := range item.Parents {
-				// only handle parents that are in the requested dirs list
-				i := sort.SearchStrings(dirs, parent)
-				if i == len(dirs) || dirs[i] != parent {
-					continue
-				}
-				remote := path.Join(paths[i], item.Name)
-				entry, err := f.itemToDirEntry(remote, item)
-				if err != nil {
-					iErr = err
-					return true
+			parentPath := ""
+			if len(item.Parents) > 0 {
+				p, ok := f.dirCache.GetInv(item.Parents[0])
+				if ok {
+					parentPath = p
 				}
+			}
+			remote := path.Join(parentPath, item.Name)
+			entry, err := f.itemToDirEntry(remote, item)
+			if err != nil {
+				iErr = err
+				return true
+			}
 
-				err = cb(entry)
-				if err != nil {
-					iErr = err
-					return true
-				}
+			err = cb(entry)
+			if err != nil {
+				iErr = err
+				return true
 			}
 			return false
 		})
@@ -1522,44 +1418,30 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
-	if directoryID == "root" {
-		var info *drive.File
-		err = f.pacer.CallNoRetry(func() (bool, error) {
-			info, err = f.svc.Files.Get("root").
-				Fields("id").
-				SupportsTeamDrives(f.isTeamDrive).
-				Do()
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return err
-		}
-		directoryID = info.Id
-	}
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
-	in := make(chan listREntry, inputBuffer)
+	in := make(chan string, inputBuffer)
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
-	overflow := []listREntry{}
+	overfflow := []string{}
 
 	cb := func(entry fs.DirEntry) error {
 		mu.Lock()
 		defer mu.Unlock()
 		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
 			select {
-			case in <- listREntry{d.ID(), d.Remote()}:
+			case in <- d.ID():
 				wg.Add(1)
 			default:
-				overflow = append(overflow, listREntry{d.ID(), d.Remote()})
+				overfflow = append(overfflow, d.ID())
 			}
 		}
 		return list.Add(entry)
 	}
 
 	wg.Add(1)
-	in <- listREntry{directoryID, dir}
+	in <- directoryID
 
 	for i := 0; i < fs.Config.Checkers; i++ {
 		go f.listRRunner(&wg, in, out, cb, grouping)
@@ -1568,18 +1450,18 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 		// wait until the all directories are processed
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
-		for len(overflow) > 0 {
+		for len(overfflow) > 0 {
 			mu.Lock()
-			l := len(overflow)
-			// only fill half of the channel to prevent entries beeing put into overflow again
+			l := len(overfflow)
+			// only fill half of the channel to prevent entries beeing put into overfflow again
 			if l > inputBuffer/2 {
 				l = inputBuffer / 2
 			}
 			wg.Add(l)
-			for _, d := range overflow[:l] {
+			for _, d := range overfflow[:l] {
 				in <- d
 			}
-			overflow = overflow[l:]
+			overfflow = overfflow[l:]
 			mu.Unlock()
 
 			// wait again for the completion of all directories
@@ -1770,14 +1652,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
+				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.rmdir(srcDir.ID(), true)
 		if err != nil {
-			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1896,24 +1778,16 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		remote = remote[:len(remote)-len(ext)]
 	}
 
-	// Look to see if there is an existing object
-	existingObject, _ := f.NewObject(remote)
-
 	createInfo, err := f.createFileInfo(remote, src.ModTime())
 	if err != nil {
 		return nil, err
 	}
 
-	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
-	if err != nil {
-		return nil, err
-	}
-
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Copy(srcObj.id, createInfo).
 			Fields(partialFields).
-			SupportsTeamDrives(supportTeamDrives).
+			SupportsTeamDrives(f.isTeamDrive).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
 			Do()
 		return shouldRetry(err)
@@ -1921,17 +1795,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	newObject, err := f.newObjectWithInfo(remote, info)
-	if err != nil {
-		return nil, err
-	}
-	if existingObject != nil {
-		err = existingObject.Remove()
-		if err != nil {
-			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
-		}
-	}
-	return newObject, nil
+	return f.newObjectWithInfo(remote, info)
 }
 
 // Purge deletes all the files and the container
@@ -2057,11 +1921,6 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
-	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
-	if err != nil {
-		return nil, err
-	}
-
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
@@ -2069,7 +1928,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
 			Fields(partialFields).
-			SupportsTeamDrives(supportTeamDrives).
+			SupportsTeamDrives(f.isTeamDrive).
 			Do()
 		return shouldRetry(err)
 	})
@@ -2080,20 +1939,6 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, info)
 }
 
-// ShouldSupportTeamDrives returns the request should support TeamDrives
-func (f *Fs) ShouldSupportTeamDrives(src fs.Object) (bool, error) {
-	srcIsTeamDrive := false
-	if srcFs, ok := src.Fs().(*Fs); ok {
-		srcIsTeamDrive = srcFs.isTeamDrive
-	}
-
-	if f.isTeamDrive {
-		return true, nil
-	}
-
-	return srcIsTeamDrive, nil
-}
-
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(remote string) (link string, err error) {
 	id, err := f.dirCache.FindDir(remote, false)
@@ -2233,7 +2078,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // ChangeNotify calls the passed function with a path that has had changes.
 // If the implementation uses polling, it should adhere to the given interval.
 //
-// Automatically restarts itself in case of unexpected behavior of the remote.
+// Automatically restarts itself in case of unexpected behaviour of the remote.
 //
 // Close the returned channel to stop being notified.
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
@@ -2340,13 +2185,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPage
 
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
-					for _, parent := range change.File.Parents {
-						if parentPath, ok := f.dirCache.GetInv(parent); ok {
-							// and append the drive file name to compute the full file name
-							newPath := path.Join(parentPath, change.File.Name)
-							// this will now clear the actual file too
-							pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
-						}
+					if parentPath, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
+						// and append the drive file name to compute the full file name
+						newPath := path.Join(parentPath, change.File.Name)
+						// this will now clear the actual file too
+						pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
 					}
 				} else { // a true root object that is changed
 					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
@@ -2526,10 +2369,6 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
 		return req, nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
-	if o.bytes == 0 {
-		// Don't supply range requests for 0 length objects as they always fail
-		delete(req.Header, "Range")
-	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {
@@ -2642,32 +2481,16 @@ func (o *documentObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err e
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
-	var offset, end int64 = 0, -1
-	var newOptions = options[:0]
 	for _, o := range options {
-		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
-		// So do a subset of them manually
-		switch x := o.(type) {
-		case *fs.RangeOption:
-			offset, end = x.Start, x.End
-		case *fs.SeekOption:
-			offset, end = x.Offset, -1
-		default:
-			newOptions = append(newOptions, o)
+		if _, ok := o.(*fs.RangeOption); ok {
+			return nil, errors.New("partial downloads are not supported while exporting Google Documents")
 		}
 	}
-	options = newOptions
-	if offset != 0 {
-		return nil, errors.New("partial downloads are not supported while exporting Google Documents")
-	}
 	in, err = o.baseObject.open(o.url, options...)
 	if in != nil {
 		in = &openDocumentFile{o: o, in: in}
 	}
-	if end >= 0 {
-		in = readers.NewLimitedReadCloser(in, end-offset+1)
-	}
 	return
 }
 func (o *linkObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
@@ -2733,9 +2556,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
-	if err != nil {
-		return err
-	}
 	switch newO := newO.(type) {
 	case *Object:
 		*o = *newO
@@ -2774,9 +2594,6 @@ func (o *documentObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.O
 	remote = remote[:len(remote)-o.extLen]
 
 	newO, err := o.fs.newObjectWithInfo(remote, info)
-	if err != nil {
-		return err
-	}
 	switch newO := newO.(type) {
 	case *documentObject:
 		*o = *newO

backend/drive/drive_internal_test.go

@@ -1,3 +1,5 @@
+// +build go1.9
+
 package drive
 
 import (

backend/drive/drive_test.go

@@ -1,5 +1,7 @@
 // Test Drive filesystem interface
 
+// +build go1.9
+
 package drive
 
 import (

backend/drive/drive_unsupported.go

@@ -0,0 +1,6 @@
+// Build for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build !go1.9
+
+package drive

backend/drive/upload.go

@@ -8,6 +8,8 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
+// +build go1.9
+
 package drive
 
 import (
@@ -183,7 +185,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionally on
+	// any other 2xx codes, so we parse it unconditionaly on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err

backend/dropbox/dropbox.go

@@ -130,8 +130,8 @@ Any files larger than this will be uploaded in chunks of this size.
 Note that chunks are buffered in memory (one at a time) so rclone can
 deal with retries.  Setting this larger will increase the speed
 slightly (at most 10%% for 128MB in tests) at the cost of using more
-memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
-			Default:  defaultChunkSize,
+memory.  It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}, {
 			Name:     "impersonate",
@@ -160,7 +160,7 @@ type Fs struct {
 	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *fs.Pacer      // To pace the API calls
+	pacer          *pacer.Pacer   // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 }
 
@@ -209,12 +209,12 @@ func shouldRetry(err error) (bool, error) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
 			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
-			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
+			time.Sleep(time.Duration(e.RateLimitError.RetryAfter) * time.Second)
 		}
 		return true, err
 	}
-	// Keep old behavior for backward compatibility
-	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") || baseErrString == "" {
+	// Keep old behaviour for backward compatibility
+	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
 		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
@@ -239,7 +239,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -273,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	config := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo

backend/ftp/ftp.go

@@ -2,7 +2,6 @@
 package ftp
 
 import (
-	"crypto/tls"
 	"io"
 	"net/textproto"
 	"os"
@@ -16,7 +15,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -47,20 +45,6 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
-			}, {
-				Name:    "tls",
-				Help:    "Use FTP over TLS (Implicit)",
-				Default: false,
-			}, {
-				Name:     "concurrency",
-				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-				Default:  0,
-				Advanced: true,
-			}, {
-				Name:     "no_check_certificate",
-				Help:     "Do not verify the TLS certificate of the server",
-				Default:  false,
-				Advanced: true,
 			},
 		},
 	})
@@ -68,13 +52,10 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string `config:"host"`
-	User              string `config:"user"`
-	Pass              string `config:"pass"`
-	Port              string `config:"port"`
-	TLS               bool   `config:"tls"`
-	Concurrency       int    `config:"concurrency"`
-	SkipVerifyTLSCert bool   `config:"no_check_certificate"`
+	Host string `config:"host"`
+	User string `config:"user"`
+	Pass string `config:"pass"`
+	Port string `config:"port"`
 }
 
 // Fs represents a remote FTP server
@@ -89,7 +70,6 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
-	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -132,15 +112,7 @@ func (f *Fs) Features() *fs.Features {
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
-	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
-	if f.opt.TLS {
-		tlsConfig := &tls.Config{
-			ServerName:         f.opt.Host,
-			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
-	}
-	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
+	c, err := ftp.DialTimeout(f.dialAddr, fs.Config.ConnectTimeout)
 	if err != nil {
 		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
 		return nil, errors.Wrap(err, "ftpConnection Dial")
@@ -156,9 +128,6 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		f.tokens.Get()
-	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -178,9 +147,6 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		defer f.tokens.Put()
-	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -200,7 +166,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	f.poolMu.Unlock()
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
@@ -223,11 +189,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	}
 
 	dialAddr := opt.Host + ":" + port
-	protocol := "ftp://"
-	if opt.TLS {
-		protocol = "ftps://"
-	}
-	u := protocol + path.Join(dialAddr+"/", root)
+	u := "ftp://" + path.Join(dialAddr+"/", root)
 	f := &Fs{
 		name:     name,
 		root:     root,
@@ -236,7 +198,6 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
-		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -369,36 +330,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
 	}
-
-	var listErr error
-	var files []*ftp.Entry
-
-	resultchan := make(chan []*ftp.Entry, 1)
-	errchan := make(chan error, 1)
-	go func() {
-		result, err := c.List(path.Join(f.root, dir))
-		f.putFtpConnection(&c, err)
-		if err != nil {
-			errchan <- err
-			return
-		}
-		resultchan <- result
-	}()
-
-	// Wait for List for up to Timeout seconds
-	timer := time.NewTimer(fs.Config.Timeout)
-	select {
-	case listErr = <-errchan:
-		timer.Stop()
-		return nil, translateErrorDir(listErr)
-	case files = <-resultchan:
-		timer.Stop()
-	case <-timer.C:
-		// if timer fired assume no error but connection dead
-		fs.Errorf(f, "Timeout when waiting for List")
-		return nil, errors.New("Timeout when waiting for List")
+	files, err := c.List(path.Join(f.root, dir))
+	f.putFtpConnection(&c, err)
+	if err != nil {
+		return nil, translateErrorDir(err)
 	}
-
 	// Annoyingly FTP returns success for a directory which
 	// doesn't exist, so check it really doesn't exist if no
 	// entries found.

backend/googlecloudstorage/googlecloudstorage.go

@@ -1,4 +1,7 @@
 // Package googlecloudstorage provides an interface to Google Cloud Storage
+
+// +build go1.9
+
 package googlecloudstorage
 
 /*
@@ -13,7 +16,6 @@ FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 erro
 */
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -43,8 +45,6 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
-
-	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
 )
 
@@ -144,22 +144,6 @@ func init() {
 				Value: "publicReadWrite",
 				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",
 			}},
-		}, {
-			Name: "bucket_policy_only",
-			Help: `Access checks should use bucket-level IAM policies.
-
-If you want to upload objects to a bucket with Bucket Policy Only set
-then you will need to set this.
-
-When it is set, rclone:
-
-- ignores ACLs set on buckets
-- ignores ACLs set on objects
-- creates buckets with Bucket Policy Only set
-
-Docs: https://cloud.google.com/storage/docs/bucket-policy-only
-`,
-			Default: false,
 		}, {
 			Name: "location",
 			Help: "Location for the newly created buckets.",
@@ -178,36 +162,21 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "asia-east1",
 				Help:  "Taiwan.",
-			}, {
-				Value: "asia-east2",
-				Help:  "Hong Kong.",
 			}, {
 				Value: "asia-northeast1",
 				Help:  "Tokyo.",
-			}, {
-				Value: "asia-south1",
-				Help:  "Mumbai.",
 			}, {
 				Value: "asia-southeast1",
 				Help:  "Singapore.",
 			}, {
 				Value: "australia-southeast1",
 				Help:  "Sydney.",
-			}, {
-				Value: "europe-north1",
-				Help:  "Finland.",
 			}, {
 				Value: "europe-west1",
 				Help:  "Belgium.",
 			}, {
 				Value: "europe-west2",
 				Help:  "London.",
-			}, {
-				Value: "europe-west3",
-				Help:  "Frankfurt.",
-			}, {
-				Value: "europe-west4",
-				Help:  "Netherlands.",
 			}, {
 				Value: "us-central1",
 				Help:  "Iowa.",
@@ -220,9 +189,6 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "us-west1",
 				Help:  "Oregon.",
-			}, {
-				Value: "us-west2",
-				Help:  "California.",
 			}},
 		}, {
 			Name: "storage_class",
@@ -257,7 +223,6 @@ type Options struct {
 	ServiceAccountCredentials string `config:"service_account_credentials"`
 	ObjectACL                 string `config:"object_acl"`
 	BucketACL                 string `config:"bucket_acl"`
-	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
 	Location                  string `config:"location"`
 	StorageClass              string `config:"storage_class"`
 }
@@ -273,7 +238,7 @@ type Fs struct {
 	bucket     string           // the bucket we are working on
 	bucketOKMu sync.Mutex       // mutex to protect bucket OK
 	bucketOK   bool             // true if we have created the bucket
-	pacer      *fs.Pacer        // To pace the API calls
+	pacer      *pacer.Pacer     // To pace the API calls
 }
 
 // Object describes a storage object
@@ -317,7 +282,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whether a given err rates being retried
+// shouldRetry determines whehter a given err rates being retried
 func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
@@ -365,7 +330,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 
@@ -398,11 +363,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
 		if err != nil {
-			ctx := context.Background()
-			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
-			}
+			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
 		}
 	}
 
@@ -416,7 +377,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		opt:    *opt,
-		pacer:  fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -730,19 +691,8 @@ func (f *Fs) Mkdir(dir string) (err error) {
 		Location:     f.opt.Location,
 		StorageClass: f.opt.StorageClass,
 	}
-	if f.opt.BucketPolicyOnly {
-		bucket.IamConfiguration = &storage.BucketIamConfiguration{
-			BucketPolicyOnly: &storage.BucketIamConfigurationBucketPolicyOnly{
-				Enabled: true,
-			},
-		}
-	}
 	err = f.pacer.Call(func() (bool, error) {
-		insertBucket := f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket)
-		if !f.opt.BucketPolicyOnly {
-			insertBucket.PredefinedAcl(f.opt.BucketACL)
-		}
-		_, err = insertBucket.Do()
+		_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -1003,15 +953,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		Bucket:      o.fs.bucket,
 		Name:        o.fs.root + o.remote,
 		ContentType: fs.MimeType(src),
+		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		insertObject := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
-		if !o.fs.opt.BucketPolicyOnly {
-			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
-		}
-		newObject, err = insertObject.Do()
+		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,5 +1,7 @@
 // Test GoogleCloudStorage filesystem interface
 
+// +build go1.9
+
 package googlecloudstorage_test
 
 import (

backend/googlecloudstorage/googlecloudstorage_unsupported.go

@@ -0,0 +1,6 @@
+// Build for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build !go1.9
+
+package googlecloudstorage

backend/http/http.go

@@ -6,7 +6,6 @@ package http
 
 import (
 	"io"
-	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -41,26 +40,7 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "https://example.com",
 				Help:  "Connect to example.com",
-			}, {
-				Value: "https://user:pass@example.com",
-				Help:  "Connect to example.com using a username and password",
 			}},
-		}, {
-			Name: "no_slash",
-			Help: `Set this if the site doesn't end directories with /
-
-Use this if your target website does not use / on the end of
-directories.
-
-A / on the end of a path is how rclone normally tells the difference
-between files and directories.  If this flag is set, then rclone will
-treat all files with Content-Type: text/html as directories and read
-URLs from them rather than downloading them.
-
-Note that this may cause rclone to confuse genuine HTML files with
-directories.`,
-			Default:  false,
-			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -69,7 +49,6 @@ directories.`,
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
-	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -269,7 +248,7 @@ func parseName(base *url.URL, name string) (string, error) {
 	}
 	// calculate the name relative to the base
 	name = u.Path[len(base.Path):]
-	// mustn't be empty
+	// musn't be empty
 	if name == "" {
 		return "", errNameIsEmpty
 	}
@@ -288,20 +267,14 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var (
-		walk func(*html.Node)
-		seen = make(map[string]struct{})
-	)
+	var walk func(*html.Node)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						if _, found := seen[name]; !found {
-							names = append(names, name)
-							seen[name] = struct{}{}
-						}
+						names = append(names, name)
 					}
 					break
 				}
@@ -326,16 +299,14 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil {
-		defer fs.CheckClose(res.Body, &err)
-		if res.StatusCode == http.StatusNotFound {
-			return nil, fs.ErrorDirNotFound
-		}
+	if err == nil && res.StatusCode == http.StatusNotFound {
+		return nil, fs.ErrorDirNotFound
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
+	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -379,16 +350,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			switch err = file.stat(); err {
-			case nil:
-				entries = append(entries, file)
-			case fs.ErrorNotAFile:
-				// ...found a directory not a file
-				dir := fs.NewDir(remote, timeUnset)
-				entries = append(entries, dir)
-			default:
+			if err = file.stat(); err != nil {
 				fs.Debugf(remote, "skipping because of error: %v", err)
+				continue
 			}
+			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -464,16 +430,6 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
-	// If NoSlash is set then check ContentType to see if it is a directory
-	if o.fs.opt.NoSlash {
-		mediaType, _, err := mime.ParseMediaType(o.contentType)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
-		}
-		if mediaType == "text/html" {
-			return fs.ErrorNotAFile
-		}
-	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -1,3 +1,5 @@
+// +build go1.8
+
 package http
 
 import (
@@ -63,7 +65,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
+func testListRoot(t *testing.T, f fs.Fs) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -91,29 +93,15 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	if noSlash {
-		assert.Equal(t, int64(-1), e.Size())
-		_, ok = e.(fs.Directory)
-		assert.True(t, ok)
-	} else {
-		assert.Equal(t, int64(41), e.Size())
-		_, ok = e.(*Object)
-		assert.True(t, ok)
-	}
+	assert.Equal(t, int64(7), e.Size())
+	_, ok = e.(*Object)
+	assert.True(t, ok)
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f, false)
-}
-
-func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
-	f.(*Fs).opt.NoSlash = true
-	defer tidy()
-
-	testListRoot(t, f, true)
+	testListRoot(t, f)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -206,7 +194,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f, false)
+	testListRoot(t, f)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-<a href="two.html/file.txt">file.txt</a>
+potato

backend/hubic/hubic.go

@@ -9,10 +9,8 @@ package hubic
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/swift"
@@ -126,9 +124,7 @@ func (f *Fs) getCredentials() (err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
-		return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
+		return errors.Errorf("failed to get credentials: %s", resp.Status)
 	}
 	decoder := json.NewDecoder(resp.Body)
 	var result credentials

backend/hubic/hubic_test.go

@@ -11,9 +11,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:          "TestHubic:",
-		NilObject:           (*hubic.Object)(nil),
-		SkipFsCheckWrap:     true,
-		SkipObjectCheckWrap: true,
+		RemoteName: "TestHubic:",
+		NilObject:  (*hubic.Object)(nil),
 	})
 }

backend/jottacloud/api/types.go

@@ -9,10 +9,7 @@ import (
 )
 
 const (
-	// default time format for almost all request and responses
 	timeFormat = "2006-01-02-T15:04:05Z0700"
-	// the API server seems to use a different format
-	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
 )
 
 // Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -43,9 +40,6 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 // Return Time string in Jottacloud format
 func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 
-// APIString returns Time string in Jottacloud API format
-func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
-
 // Flag is a hacky type for checking if an attribute is present
 type Flag bool
 
@@ -64,15 +58,6 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	return attr, errors.New("unimplemented")
 }
 
-// TokenJSON is the struct representing the HTTP response from OAuth2
-// providers returning a token in JSON form.
-type TokenJSON struct {
-	AccessToken  string `json:"access_token"`
-	TokenType    string `json:"token_type"`
-	RefreshToken string `json:"refresh_token"`
-	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
-}
-
 /*
 GET http://www.jottacloud.com/JFS/<account>
 
@@ -280,43 +265,3 @@ func (e *Error) Error() string {
 	}
 	return out
 }
-
-// AllocateFileRequest to prepare an upload to Jottacloud
-type AllocateFileRequest struct {
-	Bytes    int64  `json:"bytes"`
-	Created  string `json:"created"`
-	Md5      string `json:"md5"`
-	Modified string `json:"modified"`
-	Path     string `json:"path"`
-}
-
-// AllocateFileResponse for upload requests
-type AllocateFileResponse struct {
-	Name      string `json:"name"`
-	Path      string `json:"path"`
-	State     string `json:"state"`
-	UploadID  string `json:"upload_id"`
-	UploadURL string `json:"upload_url"`
-	Bytes     int64  `json:"bytes"`
-	ResumePos int64  `json:"resume_pos"`
-}
-
-// UploadResponse after an upload
-type UploadResponse struct {
-	Name      string      `json:"name"`
-	Path      string      `json:"path"`
-	Kind      string      `json:"kind"`
-	ContentID string      `json:"content_id"`
-	Bytes     int64       `json:"bytes"`
-	Md5       string      `json:"md5"`
-	Created   int64       `json:"created"`
-	Modified  int64       `json:"modified"`
-	Deleted   interface{} `json:"deleted"`
-	Mime      string      `json:"mime"`
-}
-
-// DeviceRegistrationResponse is the response to registering a device
-type DeviceRegistrationResponse struct {
-	ClientID     string `json:"client_id"`
-	ClientSecret string `json:"client_secret"`
-}

backend/jottacloud/jottacloud.go

@@ -7,8 +7,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -28,204 +26,48 @@ import (
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
-	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
-	"golang.org/x/oauth2"
 )
 
 // Globals
 const (
-	minSleep                    = 10 * time.Millisecond
-	maxSleep                    = 2 * time.Second
-	decayConstant               = 2 // bigger for slower decay, exponential
-	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Archive"
-	rootURL                     = "https://www.jottacloud.com/jfs/"
-	apiURL                      = "https://api.jottacloud.com/files/v1/"
-	baseURL                     = "https://www.jottacloud.com/"
-	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
-	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
-	cachePrefix                 = "rclone-jcmd5-"
-	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
-	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
-	configUsername              = "user"
-	configClientID              = "client_id"
-	configClientSecret          = "client_secret"
-	configDevice                = "device"
-	configMountpoint            = "mountpoint"
-	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-)
-
-var (
-	// Description of how to auth for this app for a personal account
-	oauthConfig = &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
-		},
-		RedirectURL: oauthutil.RedirectLocalhostURL,
-	}
+	minSleep          = 10 * time.Millisecond
+	maxSleep          = 2 * time.Second
+	decayConstant     = 2 // bigger for slower decay, exponential
+	defaultDevice     = "Jotta"
+	defaultMountpoint = "Sync"
+	rootURL           = "https://www.jottacloud.com/jfs/"
+	apiURL            = "https://api.jottacloud.com"
+	shareURL          = "https://www.jottacloud.com/"
+	cachePrefix       = "rclone-jcmd5-"
 )
 
 // Register with Fs
 func init() {
-	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
 		Description: "JottaCloud",
 		NewFs:       NewFs,
-		Config: func(name string, m configmap.Mapper) {
-			tokenString, ok := m.Get("token")
-			if ok && tokenString != "" {
-				fmt.Printf("Already have a token - refresh?\n")
-				if !config.Confirm() {
-					return
-				}
-			}
-
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
-
-			// ask if we should create a device specifc token: https://github.com/ncw/rclone/issues/2995
-			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
-			if config.Confirm() {
-				// random generator to generate random device names
-				seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
-				randonDeviceNamePartLength := 21
-				randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
-				for i := range randomDeviceNamePart {
-					randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
-				}
-				randomDeviceName := "rclone-" + string(randomDeviceNamePart)
-				fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
-
-				values := url.Values{}
-				values.Set("device_id", randomDeviceName)
-
-				// all information comes from https://github.com/ttyridal/aiojotta/wiki/Jotta-protocol-3.-Authentication#token-authentication
-				opts := rest.Opts{
-					Method:       "POST",
-					RootURL:      registerURL,
-					ContentType:  "application/x-www-form-urlencoded",
-					ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
-					Parameters:   values,
-				}
-
-				var deviceRegistration api.DeviceRegistrationResponse
-				_, err := srv.CallJSON(&opts, nil, &deviceRegistration)
-				if err != nil {
-					log.Fatalf("Failed to register device: %v", err)
-				}
-
-				m.Set(configClientID, deviceRegistration.ClientID)
-				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
-				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
-			}
-
-			clientID, ok := m.Get(configClientID)
-			if !ok {
-				clientID = rcloneClientID
-			}
-			clientSecret, ok := m.Get(configClientSecret)
-			if !ok {
-				clientSecret = rcloneEncryptedClientSecret
-			}
-			oauthConfig.ClientID = clientID
-			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
-
-			username, ok := m.Get(configUsername)
-			if !ok {
-				log.Fatalf("No username defined")
-			}
-			password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
-
-			// prepare out token request with username and password
-			values := url.Values{}
-			values.Set("grant_type", "PASSWORD")
-			values.Set("password", password)
-			values.Set("username", username)
-			values.Set("client_id", oauthConfig.ClientID)
-			values.Set("client_secret", oauthConfig.ClientSecret)
-			opts := rest.Opts{
-				Method:      "POST",
-				RootURL:     oauthConfig.Endpoint.AuthURL,
-				ContentType: "application/x-www-form-urlencoded",
-				Parameters:  values,
-			}
-
-			var jsonToken api.TokenJSON
-			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
-			if err != nil {
-				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
-				if resp != nil {
-					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
-						fmt.Printf("Enter verification code> ")
-						authCode := config.ReadLine()
-						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
-						opts.ExtraHeaders = make(map[string]string)
-						opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-						resp, err = srv.CallJSON(&opts, nil, &jsonToken)
-					}
-				}
-				if err != nil {
-					log.Fatalf("Failed to get resource token: %v", err)
-				}
-			}
-
-			var token oauth2.Token
-			token.AccessToken = jsonToken.AccessToken
-			token.RefreshToken = jsonToken.RefreshToken
-			token.TokenType = jsonToken.TokenType
-			token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
-
-			// finally save them in the config
-			err = oauthutil.PutToken(name, m, &token, true)
-			if err != nil {
-				log.Fatalf("Error while saving token: %s", err)
-			}
-
-			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
-			if config.Confirm() {
-				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
-				if err != nil {
-					log.Fatalf("Failed to load oAuthClient: %s", err)
-				}
-
-				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
-
-				acc, err := getAccountInfo(srv, username)
-				if err != nil {
-					log.Fatalf("Error getting devices: %s", err)
-				}
-				fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
-				var deviceNames []string
-				for i := range acc.Devices {
-					deviceNames = append(deviceNames, acc.Devices[i].Name)
-				}
-				result := config.Choose("Devices", deviceNames, nil, false)
-				m.Set(configDevice, result)
-
-				dev, err := getDeviceInfo(srv, path.Join(username, result))
-				if err != nil {
-					log.Fatalf("Error getting Mountpoint: %s", err)
-				}
-				if len(dev.MountPoints) == 0 {
-					log.Fatalf("No Mountpoints found for this device.")
-				}
-				fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
-				var mountpointNames []string
-				for i := range dev.MountPoints {
-					mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
-				}
-				result = config.Choose("Mountpoints", mountpointNames, nil, false)
-				m.Set(configMountpoint, result)
-			}
-		},
 		Options: []fs.Option{{
-			Name: configUsername,
-			Help: "User Name:",
+			Name: "user",
+			Help: "User Name",
+		}, {
+			Name:       "pass",
+			Help:       "Password.",
+			IsPassword: true,
+		}, {
+			Name:     "mountpoint",
+			Help:     "The mountpoint to use.",
+			Required: true,
+			Examples: []fs.OptionExample{{
+				Value: "Sync",
+				Help:  "Will be synced by the official client.",
+			}, {
+				Value: "Archive",
+				Help:  "Archive",
+			}},
 		}, {
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -241,11 +83,6 @@ func init() {
 			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:     "upload_resume_limit",
-			Help:     "Files bigger than this can be resumed if the upload fail's.",
-			Default:  fs.SizeSuffix(10 * 1024 * 1024),
-			Advanced: true,
 		}},
 	})
 }
@@ -253,26 +90,23 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
-	Device             string        `config:"device"`
+	Pass               string        `config:"pass"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
 	Unlink             bool          `config:"unlink"`
-	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
 }
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name         string
-	root         string
-	user         string
-	opt          Options
-	features     *fs.Features
-	endpointURL  string
-	srv          *rest.Client
-	apiSrv       *rest.Client
-	pacer        *fs.Pacer
-	tokenRenewer *oauthutil.Renew // renew the token on expiry
+	name        string
+	root        string
+	user        string
+	opt         Options
+	features    *fs.Features
+	endpointURL string
+	srv         *rest.Client
+	pacer       *pacer.Pacer
 }
 
 // Object describes a jottacloud object
@@ -361,31 +195,18 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
 	return &result, nil
 }
 
-// getAccountInfo queries general information about the account.
-// Takes rest.Client and username as parameter to be easily usable
-// during config
-func getAccountInfo(srv *rest.Client, username string) (info *api.AccountInfo, err error) {
+// getAccountInfo retrieves account information
+func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
-		Path:   urlPathEscape(username),
+		Path:   urlPathEscape(f.user),
 	}
 
-	_, err = srv.CallXML(&opts, nil, &info)
-	if err != nil {
-		return nil, err
-	}
-
-	return info, nil
-}
-
-// getDeviceInfo queries Information about a jottacloud device
-func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) {
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   urlPathEscape(path),
-	}
-
-	_, err = srv.CallXML(&opts, nil, &info)
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallXML(&opts, nil, &info)
+		return shouldRetry(resp, err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -394,18 +215,12 @@ func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err er
 }
 
 // setEndpointUrl reads the account id and generates the API endpoint URL
-func (f *Fs) setEndpointURL() (err error) {
-	info, err := getAccountInfo(f.srv, f.user)
+func (f *Fs) setEndpointURL(mountpoint string) (err error) {
+	info, err := f.getAccountInfo()
 	if err != nil {
 		return errors.Wrap(err, "failed to get endpoint url")
 	}
-	if f.opt.Device == "" {
-		f.opt.Device = defaultDevice
-	}
-	if f.opt.Mountpoint == "" {
-		f.opt.Mountpoint = defaultMountpoint
-	}
-	f.endpointURL = urlPathEscape(path.Join(info.Username, f.opt.Device, f.opt.Mountpoint))
+	f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
 	return nil
 }
 
@@ -446,29 +261,6 @@ func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
 
-// Jottacloud requires the grant_type 'refresh_token' string
-// to be uppercase and throws a 400 Bad Request if we use the
-// lower case used by the oauth2 module
-//
-// This filter catches all refresh requests, reads the body,
-// changes the case and then sends it on
-func grantTypeFilter(req *http.Request) {
-	if tokenURL == req.URL.String() {
-		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
-		if err != nil {
-			return
-		}
-		_ = req.Body.Close()
-
-		// make the refresh token upper case
-		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
-
-		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
-	}
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -481,40 +273,25 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	clientID, ok := m.Get(configClientID)
-	if !ok {
-		clientID = rcloneClientID
-	}
-	clientSecret, ok := m.Get(configClientSecret)
-	if !ok {
-		clientSecret = rcloneEncryptedClientSecret
-	}
-	oauthConfig.ClientID = clientID
-	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+	user := config.FileGet(name, "user")
+	pass := config.FileGet(name, "pass")
 
-	// the oauth client for the api servers needs
-	// a filter to fix the grant_type issues (see above)
-	baseClient := fshttp.NewClient(fs.Config)
-	if do, ok := baseClient.Transport.(interface {
-		SetRequestFilter(f func(req *http.Request))
-	}); ok {
-		do.SetRequestFilter(grantTypeFilter)
-	} else {
-		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
-	}
-	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
+	if opt.Pass != "" {
+		var err error
+		opt.Pass, err = obscure.Reveal(opt.Pass)
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't decrypt password")
+		}
 	}
 
 	f := &Fs{
-		name:   name,
-		root:   root,
-		user:   opt.User,
-		opt:    *opt,
-		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
-		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		name: name,
+		root: root,
+		user: opt.User,
+		opt:  *opt,
+		//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
+		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -522,15 +299,15 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 	}).Fill(f)
+
+	if user == "" || pass == "" {
+		return nil, errors.New("jottacloud needs user and password")
+	}
+
+	f.srv.SetUserPass(opt.User, opt.Pass)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Renew the token in the background
-	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.readMetaDataForPath("")
-		return err
-	})
-
-	err = f.setEndpointURL()
+	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
 	}
@@ -554,6 +331,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+
 	return f, nil
 }
 
@@ -570,7 +348,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData(false) // reads info and meta, returning an error
+		err = o.readMetaData() // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -618,7 +396,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	//fmt.Printf("List: %s\n", f.filePath(dir))
+	//fmt.Printf("List: %s\n", dir)
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   f.filePath(dir),
@@ -788,9 +566,6 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if f.opt.Device != "Jotta" {
-		return nil, errors.New("upload not supported for devices other than Jotta")
-	}
 	o := f.createObject(src.Remote(), src.ModTime(), src.Size())
 	return o, o.Update(in, src, options...)
 }
@@ -883,7 +658,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copies or moves directories or files depending on the method parameter
+// copyOrMoves copys or moves directories or files depending on the mthod parameter
 func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -901,6 +676,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
 	if err != nil {
 		return nil, err
 	}
+
 	return info, nil
 }
 
@@ -1048,13 +824,13 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if result.PublicSharePath == "" {
 		return "", errors.New("couldn't create public link - no link path received")
 	}
-	link = path.Join(baseURL, result.PublicSharePath)
+	link = path.Join(shareURL, result.PublicSharePath)
 	return link, nil
 }
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
-	info, err := getAccountInfo(f.srv, f.user)
+	info, err := f.getAccountInfo()
 	if err != nil {
 		return nil, err
 	}
@@ -1104,7 +880,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return 0
@@ -1120,24 +896,21 @@ func (o *Object) MimeType() string {
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.hasMetaData = true
-	o.size = info.Size
+	o.size = int64(info.Size)
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
 	o.modTime = time.Time(info.ModifiedAt)
 	return nil
 }
 
-func (o *Object) readMetaData(force bool) (err error) {
-	if o.hasMetaData && !force {
+func (o *Object) readMetaData() (err error) {
+	if o.hasMetaData {
 		return nil
 	}
 	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		return err
 	}
-	if info.Deleted {
-		return fs.ErrorObjectNotFound
-	}
 	return o.setMetaData(info)
 }
 
@@ -1146,7 +919,7 @@ func (o *Object) readMetaData(force bool) (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1194,7 +967,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
 	// we need a MD5
 	md5Hasher := md5.New()
-	// use the teeReader to write to the local file AND calculate the MD5 while doing so
+	// use the teeReader to write to the local file AND caclulate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
 
 	// nothing to clean up by default
@@ -1267,74 +1040,43 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		in = wrap(in)
 	}
 
-	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
+	var result api.JottaFile
 	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "allocate",
-		ExtraHeaders: make(map[string]string),
-	}
-	fileDate := api.Time(src.ModTime()).APIString()
-
-	// the allocate request
-	var request = api.AllocateFileRequest{
-		Bytes:    size,
-		Created:  fileDate,
-		Modified: fileDate,
-		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
+		Method:        "POST",
+		Path:          o.filePath(),
+		Body:          in,
+		ContentType:   fs.MimeType(src),
+		ContentLength: &size,
+		ExtraHeaders:  make(map[string]string),
+		Parameters:    url.Values{},
 	}
 
-	// send it
-	var response api.AllocateFileResponse
+	opts.ExtraHeaders["JMd5"] = md5String
+	opts.Parameters.Set("cphash", md5String)
+	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
+	// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
+	opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
+
+	// Parameters observed in other implementations
+	//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
+	//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
+	//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
+	//opts.ExtraHeaders["jx_csid"] = ""
+	//opts.ExtraHeaders["jx_lisence"] = ""
+
+	opts.Parameters.Set("umode", "nomultipart")
+
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
+		resp, err = o.fs.srv.CallXML(&opts, nil, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
-	if response.State != "COMPLETED" {
-		// how much do we still have to upload?
-		remainingBytes := size - response.ResumePos
-		opts = rest.Opts{
-			Method:        "POST",
-			RootURL:       response.UploadURL,
-			ContentLength: &remainingBytes,
-			ContentType:   "application/octet-stream",
-			Body:          in,
-			ExtraHeaders:  make(map[string]string),
-		}
-		if response.ResumePos != 0 {
-			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
-		}
-
-		// copy the already uploaded bytes into the trash :)
-		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
-		if err != nil {
-			return err
-		}
-
-		// send the remaining bytes
-		resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
-		if err != nil {
-			return err
-		}
-
-		// finally update the meta data
-		o.hasMetaData = true
-		o.size = result.Bytes
-		o.md5 = result.Md5
-		o.modTime = time.Unix(result.Modified/1000, 0)
-	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
-		return o.readMetaData(true)
-	}
-
-	return nil
+	// TODO: Check returned Metadata? Timeout on big uploads?
+	return o.setMetaData(&result)
 }
 
 // Remove an object

backend/jottacloud/replace.go

@@ -2,7 +2,7 @@
 Translate file names for JottaCloud adapted from OneDrive
 
 
-The following characters are JottaCloud reserved characters, and can't
+The following characters are JottaClous reserved characters, and can't
 be used in JottaCloud folder and file names.
 
   jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"

backend/koofr/koofr.go

@@ -1,589 +0,0 @@
-package koofr
-
-import (
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
-	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fs/hash"
-
-	httpclient "github.com/koofr/go-httpclient"
-	koofrclient "github.com/koofr/go-koofrclient"
-)
-
-// Register Fs with rclone
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "koofr",
-		Description: "Koofr",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "The Koofr API endpoint to use",
-				Default:  "https://app.koofr.net",
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "mountid",
-				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
-				Required: false,
-				Default:  "",
-				Advanced: true,
-			}, {
-				Name:     "user",
-				Help:     "Your Koofr user name",
-				Required: true,
-			}, {
-				Name:       "password",
-				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
-				IsPassword: true,
-				Required:   true,
-			},
-		},
-	})
-}
-
-// Options represent the configuration of the Koofr backend
-type Options struct {
-	Endpoint string `config:"endpoint"`
-	MountID  string `config:"mountid"`
-	User     string `config:"user"`
-	Password string `config:"password"`
-}
-
-// A Fs is a representation of a remote Koofr Fs
-type Fs struct {
-	name     string
-	mountID  string
-	root     string
-	opt      Options
-	features *fs.Features
-	client   *koofrclient.KoofrClient
-}
-
-// An Object on the remote Koofr Fs
-type Object struct {
-	fs     *Fs
-	remote string
-	info   koofrclient.FileInfo
-}
-
-func base(pth string) string {
-	rv := path.Base(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-func dir(pth string) string {
-	rv := path.Dir(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-// String returns a string representation of the remote Object
-func (o *Object) String() string {
-	return o.remote
-}
-
-// Remote returns the remote path of the Object, relative to Fs root
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the Object
-func (o *Object) ModTime() time.Time {
-	return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000)
-}
-
-// Size return the size of the Object in bytes
-func (o *Object) Size() int64 {
-	return o.info.Size
-}
-
-// Fs returns a reference to the Koofr Fs containing the Object
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns an MD5 hash of the Object
-func (o *Object) Hash(typ hash.Type) (string, error) {
-	if typ == hash.MD5 {
-		return o.info.Hash, nil
-	}
-	return "", nil
-}
-
-// fullPath returns full path of the remote Object (including Fs root)
-func (o *Object) fullPath() string {
-	return o.fs.fullPath(o.remote)
-}
-
-// Storable returns true if the Object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime is not supported
-func (o *Object) SetModTime(mtime time.Time) error {
-	return nil
-}
-
-// Open opens the Object for reading
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
-	var sOff, eOff int64 = 0, -1
-
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.SeekOption:
-			sOff = x.Offset
-		case *fs.RangeOption:
-			sOff = x.Start
-			eOff = x.End
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if sOff == 0 && eOff < 0 {
-		return o.fs.client.FilesGet(o.fs.mountID, o.fullPath())
-	}
-	if sOff < 0 {
-		sOff = o.Size() - eOff
-		eOff = o.Size()
-	}
-	if eOff > o.Size() {
-		eOff = o.Size()
-	}
-	span := &koofrclient.FileSpan{
-		Start: sOff,
-		End:   eOff,
-	}
-	return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span)
-}
-
-// Update updates the Object contents
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := o.fullPath()
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err := o.fs.mkdir(dirPath)
-	if err != nil {
-		return err
-	}
-	info, err := o.fs.client.FilesPutOptions(o.fs.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return err
-	}
-	o.info = *info
-	return nil
-}
-
-// Remove deletes the remote Object
-func (o *Object) Remove() error {
-	return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath())
-}
-
-// Name returns the name of the Fs
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root returns the root path of the Fs
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a string representation of the Fs
-func (f *Fs) String() string {
-	return "koofr:" + f.mountID + ":" + f.root
-}
-
-// Features returns the optional features supported by this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Precision denotes that setting modification times is not supported
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns a set of hashes are Provided by the Fs
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// fullPath constructs a full, absolute path from a Fs root relative path,
-func (f *Fs) fullPath(part string) string {
-	return path.Join("/", f.root, part)
-}
-
-// NewFs constructs a new filesystem given a root path and configuration options
-func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
-	opt := new(Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-	pass, err := obscure.Reveal(opt.Password)
-	if err != nil {
-		return nil, err
-	}
-	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
-	basicAuth := fmt.Sprintf("Basic %s",
-		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
-	client.HTTPClient.Headers.Set("Authorization", basicAuth)
-	mounts, err := client.Mounts()
-	if err != nil {
-		return nil, err
-	}
-	f := &Fs{
-		name:   name,
-		root:   root,
-		opt:    *opt,
-		client: client,
-	}
-	f.features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          false,
-		BucketBased:             false,
-		CanHaveEmptyDirectories: true,
-	}).Fill(f)
-	for _, m := range mounts {
-		if opt.MountID != "" {
-			if m.Id == opt.MountID {
-				f.mountID = m.Id
-				break
-			}
-		} else if m.IsPrimary {
-			f.mountID = m.Id
-			break
-		}
-	}
-	if f.mountID == "" {
-		if opt.MountID == "" {
-			return nil, errors.New("Failed to find primary mount")
-		}
-		return nil, errors.New("Failed to find mount " + opt.MountID)
-	}
-	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
-	if err == nil && rootFile.Type != "dir" {
-		f.root = dir(f.root)
-		err = fs.ErrorIsFile
-	} else {
-		err = nil
-	}
-	return f, err
-}
-
-// List returns a list of items in a directory
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return nil, translateErrorsDir(err)
-	}
-	entries = make([]fs.DirEntry, len(files))
-	for i, file := range files {
-		if file.Type == "dir" {
-			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
-		} else {
-			entries[i] = &Object{
-				fs:     f,
-				info:   file,
-				remote: path.Join(dir, file.Name),
-			}
-		}
-	}
-	return entries, nil
-}
-
-// NewObject creates a new remote Object for a given remote path
-func (f *Fs) NewObject(remote string) (obj fs.Object, err error) {
-	info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote))
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	if info.Type == "dir" {
-		return nil, fs.ErrorNotAFile
-	}
-	return &Object{
-		fs:     f,
-		info:   info,
-		remote: remote,
-	}, nil
-}
-
-// Put updates a remote Object
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := f.fullPath(src.Remote())
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err = f.mkdir(dirPath)
-	if err != nil {
-		return nil, err
-	}
-	info, err := f.client.FilesPutOptions(f.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	return &Object{
-		fs:     f,
-		info:   *info,
-		remote: src.Remote(),
-	}, nil
-}
-
-// PutStream updates a remote Object with a stream of unknown size
-func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(in, src, options...)
-}
-
-// isBadRequest is a predicate which holds true iff the error returned was
-// HTTP status 400
-func isBadRequest(err error) bool {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusBadRequest {
-			return true
-		}
-	}
-	return false
-}
-
-// translateErrorsDir translates koofr errors to rclone errors (for a dir
-// operation)
-func translateErrorsDir(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorDirNotFound
-		}
-	}
-	return err
-}
-
-// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
-func translateErrorsObject(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorObjectNotFound
-		}
-	}
-	return err
-}
-
-// mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
-func (f *Fs) mkdir(fullPath string) error {
-	if fullPath == "/" {
-		return nil
-	}
-	info, err := f.client.FilesInfo(f.mountID, fullPath)
-	if err == nil && info.Type == "dir" {
-		return nil
-	}
-	err = translateErrorsDir(err)
-	if err != nil && err != fs.ErrorDirNotFound {
-		return err
-	}
-	dirs := strings.Split(fullPath, "/")
-	parent := "/"
-	for _, part := range dirs {
-		if part == "" {
-			continue
-		}
-		info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part))
-		if err != nil || info.Type != "dir" {
-			err = translateErrorsDir(err)
-			if err != nil && err != fs.ErrorDirNotFound {
-				return err
-			}
-			err = f.client.FilesNewFolder(f.mountID, parent, part)
-			if err != nil && !isBadRequest(err) {
-				return err
-			}
-		}
-		parent = path.Join(parent, part)
-	}
-	return nil
-}
-
-// Mkdir creates a directory at the given remote path. Creates ancestors if
-// necessary
-func (f *Fs) Mkdir(dir string) error {
-	fullPath := f.fullPath(dir)
-	return f.mkdir(fullPath)
-}
-
-// Rmdir removes an (empty) directory at the given remote path
-func (f *Fs) Rmdir(dir string) error {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	if len(files) > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-	err = f.client.FilesDelete(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	return nil
-}
-
-// Copy copies a remote Object to the given path
-func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	err = f.client.FilesCopy((src.(*Object)).fs.mountID,
-		(src.(*Object)).fs.fullPath((src.(*Object)).remote),
-		f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	return f.NewObject(remote)
-}
-
-// Move moves a remote Object to the given path
-func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
-	srcObj := src.(*Object)
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	err = f.client.FilesMove(srcObj.fs.mountID,
-		srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	return f.NewObject(remote)
-}
-
-// DirMove moves a remote directory to the given path
-func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
-	srcFs := src.(*Fs)
-	srcFullPath := srcFs.fullPath(srcRemote)
-	dstFullPath := f.fullPath(dstRemote)
-	if srcFs.mountID == f.mountID && srcFullPath == dstFullPath {
-		return fs.ErrorDirExists
-	}
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	return nil
-}
-
-// About reports space usage (with a MB precision)
-func (f *Fs) About() (*fs.Usage, error) {
-	mount, err := f.client.MountsDetails(f.mountID)
-	if err != nil {
-		return nil, err
-	}
-	return &fs.Usage{
-		Total:   fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024),
-		Used:    fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024),
-		Trashed: nil,
-		Other:   nil,
-		Free:    fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024),
-		Objects: nil,
-	}, nil
-}
-
-// Purge purges the complete Fs
-func (f *Fs) Purge() error {
-	err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath("")))
-	return err
-}
-
-// linkCreate is a Koofr API request for creating a public link
-type linkCreate struct {
-	Path string `json:"path"`
-}
-
-// link is a Koofr API response to creating a public link
-type link struct {
-	ID               string `json:"id"`
-	Name             string `json:"name"`
-	Path             string `json:"path"`
-	Counter          int64  `json:"counter"`
-	URL              string `json:"url"`
-	ShortURL         string `json:"shortUrl"`
-	Hash             string `json:"hash"`
-	Host             string `json:"host"`
-	HasPassword      bool   `json:"hasPassword"`
-	Password         string `json:"password"`
-	ValidFrom        int64  `json:"validFrom"`
-	ValidTo          int64  `json:"validTo"`
-	PasswordRequired bool   `json:"passwordRequired"`
-}
-
-// createLink makes a Koofr API call to create a public link
-func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) {
-	linkCreate := linkCreate{
-		Path: path,
-	}
-	linkData := link{}
-
-	request := httpclient.RequestData{
-		Method:         "POST",
-		Path:           "/api/v2/mounts/" + mountID + "/links",
-		ExpectedStatus: []int{http.StatusOK, http.StatusCreated},
-		ReqEncoding:    httpclient.EncodingJSON,
-		ReqValue:       linkCreate,
-		RespEncoding:   httpclient.EncodingJSON,
-		RespValue:      &linkData,
-	}
-
-	_, err := c.Request(&request)
-	if err != nil {
-		return nil, err
-	}
-	return &linkData, nil
-}
-
-// PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(remote string) (string, error) {
-	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
-	if err != nil {
-		return "", translateErrorsDir(err)
-	}
-	return linkData.ShortURL, nil
-}

backend/koofr/koofr_test.go

@@ -1,14 +0,0 @@
-package koofr_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestKoofr:",
-	})
-}

backend/local/about_unix.go

@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
-	bs := int64(s.Bsize) // nolint: unconvert
+	bs := int64(s.Bsize)
 	usage := &fs.Usage{
 		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
 		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use

backend/local/lchtimes.go

@@ -1,20 +0,0 @@
-// +build windows plan9
-
-package local
-
-import (
-	"time"
-)
-
-const haveLChtimes = false
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	// Does nothing
-	return nil
-}

backend/local/lchtimes_unix.go

@@ -1,28 +0,0 @@
-// +build !windows,!plan9
-
-package local
-
-import (
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const haveLChtimes = true
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	var utimes [2]unix.Timespec
-	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
-	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
-	if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
-		return &os.PathError{Op: "lchtimes", Path: name, Err: e}
-	}
-	return nil
-}

backend/local/local.go

@@ -2,10 +2,10 @@
 package local
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -28,9 +28,7 @@ import (
 )
 
 // Constants
-const devUnset = 0xdeadbeefcafebabe                                       // a device id meaning it is unset
-const linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
-const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
+const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
 
 // Register with Fs
 func init() {
@@ -52,13 +50,6 @@ func init() {
 			NoPrefix: true,
 			ShortOpt: "L",
 			Advanced: true,
-		}, {
-			Name:     "links",
-			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
-			Default:  false,
-			NoPrefix: true,
-			ShortOpt: "l",
-			Advanced: true,
 		}, {
 			Name: "skip_links",
 			Help: `Don't warn about skipped symlinks.
@@ -103,13 +94,12 @@ check can be disabled with this flag.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks    bool `config:"copy_links"`
-	TranslateSymlinks bool `config:"links"`
-	SkipSymlinks      bool `config:"skip_links"`
-	NoUTFNorm         bool `config:"no_unicode_normalization"`
-	NoCheckUpdated    bool `config:"no_check_updated"`
-	NoUNC             bool `config:"nounc"`
-	OneFileSystem     bool `config:"one_file_system"`
+	FollowSymlinks bool `config:"copy_links"`
+	SkipSymlinks   bool `config:"skip_links"`
+	NoUTFNorm      bool `config:"no_unicode_normalization"`
+	NoCheckUpdated bool `config:"no_check_updated"`
+	NoUNC          bool `config:"nounc"`
+	OneFileSystem  bool `config:"one_file_system"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -131,20 +121,17 @@ type Fs struct {
 
 // Object represents a local filesystem object
 type Object struct {
-	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path - properly UTF-8 encoded - for rclone
-	path           string // The local path - may not be properly UTF-8 encoded - for OS
-	size           int64  // file metadata - always present
-	mode           os.FileMode
-	modTime        time.Time
-	hashes         map[hash.Type]string // Hashes
-	translatedLink bool                 // Is this object a translated link
+	fs      *Fs    // The Fs this object is part of
+	remote  string // The remote path - properly UTF-8 encoded - for rclone
+	path    string // The local path - may not be properly UTF-8 encoded - for OS
+	size    int64  // file metadata - always present
+	mode    os.FileMode
+	modTime time.Time
+	hashes  map[hash.Type]string // Hashes
 }
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-
 // NewFs constructs an Fs from the path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -153,9 +140,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	if opt.TranslateSymlinks && opt.FollowSymlinks {
-		return nil, errLinksAndCopyLinks
-	}
 
 	if opt.NoUTFNorm {
 		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -183,7 +167,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	if err == nil && f.isRegular(fi.Mode()) {
+	if err == nil && fi.Mode().IsRegular() {
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -192,20 +176,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
-// Determine whether a file is a 'regular' file,
-// Symlinks are regular files, only if the TranslateSymlink
-// option is in-effect
-func (f *Fs) isRegular(mode os.FileMode) bool {
-	if !f.opt.TranslateSymlinks {
-		return mode.IsRegular()
-	}
-
-	// fi.Mode().IsRegular() tests that all mode bits are zero
-	// Since symlinks are accepted, test that all other bits are zero,
-	// except the symlink bit
-	return mode&os.ModeType&^os.ModeSymlink == 0
-}
-
 // Name of the remote (as passed into NewFs)
 func (f *Fs) Name() string {
 	return f.name
@@ -226,48 +196,28 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// caseInsensitive returns whether the remote is case insensitive or not
+// caseInsenstive returns whether the remote is case insensitive or not
 func (f *Fs) caseInsensitive() bool {
 	// FIXME not entirely accurate since you can have case
-	// sensitive Fses on darwin and case insensitive Fses on linux.
+	// sensitive Fses on darwin and case insenstive Fses on linux.
 	// Should probably check but that would involve creating a
 	// file in the remote to be most accurate which probably isn't
 	// desirable.
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
-// translateLink checks whether the remote is a translated link
-// and returns a new path, removing the suffix as needed,
-// It also returns whether this is a translated link at all
-//
-// for regular files, dstPath is returned unchanged
-func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
-	return newDstPath, isTranslatedLink
-}
-
 // newObject makes a half completed Object
 //
 // if dstPath is empty then it is made from remote
 func (f *Fs) newObject(remote, dstPath string) *Object {
-	translatedLink := false
-
 	if dstPath == "" {
 		dstPath = f.cleanPath(filepath.Join(f.root, remote))
 	}
 	remote = f.cleanRemote(remote)
-
-	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for dstPath
-		dstPath, translatedLink = translateLink(remote, dstPath)
-	}
-
 	return &Object{
-		fs:             f,
-		remote:         remote,
-		path:           dstPath,
-		translatedLink: translatedLink,
+		fs:     f,
+		remote: remote,
+		path:   dstPath,
 	}
 }
 
@@ -289,11 +239,6 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
-		// Handle the odd case, that a symlink was specified by name without the link suffix
-		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
-			return nil, fs.ErrorObjectNotFound
-		}
-
 	}
 	if o.mode.IsDir() {
 		return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -317,7 +262,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-
 	dir = f.dirNames.Load(dir)
 	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
 	remote := f.cleanRemote(dir)
@@ -328,14 +272,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 
 	fd, err := os.Open(fsDirPath)
 	if err != nil {
-		isPerm := os.IsPermission(err)
-		err = errors.Wrapf(err, "failed to open directory %q", dir)
-		fs.Errorf(dir, "%v", err)
-		if isPerm {
-			accounting.Stats.Error(fserrors.NoRetryError(err))
-			err = nil // ignore error but fail sync
-		}
-		return nil, err
+		return nil, errors.Wrapf(err, "failed to open directory %q", dir)
 	}
 	defer func() {
 		cerr := fd.Close()
@@ -345,38 +282,12 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	}()
 
 	for {
-		var fis []os.FileInfo
-		if useReadDir {
-			// Windows and Plan9 read the directory entries with the stat information in which
-			// shouldn't fail because of unreadable entries.
-			fis, err = fd.Readdir(1024)
-			if err == io.EOF && len(fis) == 0 {
-				break
-			}
-		} else {
-			// For other OSes we read the names only (which shouldn't fail) then stat the
-			// individual ourselves so we can log errors but not fail the directory read.
-			var names []string
-			names, err = fd.Readdirnames(1024)
-			if err == io.EOF && len(names) == 0 {
-				break
-			}
-			if err == nil {
-				for _, name := range names {
-					namepath := filepath.Join(fsDirPath, name)
-					fi, fierr := os.Lstat(namepath)
-					if fierr != nil {
-						err = errors.Wrapf(err, "failed to read directory %q", namepath)
-						fs.Errorf(dir, "%v", fierr)
-						accounting.Stats.Error(fserrors.NoRetryError(fierr)) // fail the sync
-						continue
-					}
-					fis = append(fis, fi)
-				}
-			}
+		fis, err := fd.Readdir(1024)
+		if err == io.EOF && len(fis) == 0 {
+			break
 		}
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to read directory entry")
+			return nil, errors.Wrapf(err, "failed to read directory %q", dir)
 		}
 
 		for _, fi := range fis {
@@ -407,10 +318,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 					entries = append(entries, d)
 				}
 			} else {
-				// Check whether this link should be translated
-				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += linkSuffix
-				}
 				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
 				if err != nil {
 					return nil, err
@@ -624,7 +531,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.fs.isRegular(dstObj.mode) {
+	} else if !dstObj.mode.IsRegular() {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -743,21 +650,14 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 
 	o.fs.objectHashesMu.Lock()
 	hashes := o.hashes
-	hashValue, hashFound := o.hashes[r]
 	o.fs.objectHashesMu.Unlock()
 
-	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
-		var in io.ReadCloser
-
-		if !o.translatedLink {
-			in, err = file.Open(o.path)
-		} else {
-			in, err = o.openTranslatedLink(0, -1)
-		}
+	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
+		in, err := file.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
-		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
+		hashes, err = hash.Stream(in)
 		closeErr := in.Close()
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to read")
@@ -765,16 +665,11 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		if closeErr != nil {
 			return "", errors.Wrap(closeErr, "hash: failed to close")
 		}
-		hashValue = hashes[r]
 		o.fs.objectHashesMu.Lock()
-		if o.hashes == nil {
-			o.hashes = hashes
-		} else {
-			o.hashes[r] = hashValue
-		}
+		o.hashes = hashes
 		o.fs.objectHashesMu.Unlock()
 	}
-	return hashValue, nil
+	return hashes[r], nil
 }
 
 // Size returns the size of an object in bytes
@@ -789,12 +684,7 @@ func (o *Object) ModTime() time.Time {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	var err error
-	if o.translatedLink {
-		err = lChtimes(o.path, modTime, modTime)
-	} else {
-		err = os.Chtimes(o.path, modTime, modTime)
-	}
+	err := os.Chtimes(o.path, modTime, modTime)
 	if err != nil {
 		return err
 	}
@@ -812,7 +702,7 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
+	if mode&os.ModeSymlink != 0 {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}
@@ -853,6 +743,7 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 	}
 
 	n, err = file.in.Read(p)
+	log.Printf("*** Read result n=%d, err=%v, len(p)=%d", n, err, len(p))
 	if n > 0 {
 		// Hash routines never return an error
 		_, _ = file.hash.Write(p[:n])
@@ -873,16 +764,6 @@ func (file *localOpenFile) Close() (err error) {
 	return err
 }
 
-// Returns a ReadCloser() object that contains the contents of a symbolic link
-func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
-	// Read the link and return the destination  it as the contents of the object
-	linkdst, err := os.Readlink(o.path)
-	if err != nil {
-		return nil, err
-	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
-}
-
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
@@ -902,11 +783,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
-	// Handle a translated link
-	if o.translatedLink {
-		return o.openTranslatedLink(offset, limit)
-	}
-
 	fd, err := file.Open(o.path)
 	if err != nil {
 		return
@@ -938,19 +814,8 @@ func (o *Object) mkdirAll() error {
 	return os.MkdirAll(dir, 0777)
 }
 
-type nopWriterCloser struct {
-	*bytes.Buffer
-}
-
-func (nwc nopWriterCloser) Close() error {
-	// noop
-	return nil
-}
-
 // Update the object from in with modTime and size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	var out io.WriteCloser
-
 	hashes := hash.Supported
 	for _, option := range options {
 		switch x := option.(type) {
@@ -964,23 +829,15 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	var symlinkData bytes.Buffer
-	// If the object is a regular file, create it.
-	// If it is a translated link, just read in the contents, and
-	// then create a symlink
-	if !o.translatedLink {
-		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-		if err != nil {
-			return err
-		}
-		// Pre-allocate the file for performance reasons
-		err = preAllocate(src.Size(), f)
-		if err != nil {
-			fs.Debugf(o, "Failed to pre-allocate: %v", err)
-		}
-		out = f
-	} else {
-		out = nopWriterCloser{&symlinkData}
+	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return err
+	}
+
+	// Pre-allocate the file for performance reasons
+	err = preAllocate(src.Size(), out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
 
 	// Calculate the hash of the object we are reading as we go along
@@ -995,26 +852,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if err == nil {
 		err = closeErr
 	}
-
-	if o.translatedLink {
-		if err == nil {
-			// Remove any current symlink or file, if one exists
-			if _, err := os.Lstat(o.path); err == nil {
-				if removeErr := os.Remove(o.path); removeErr != nil {
-					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
-					return removeErr
-				}
-			}
-			// Use the contents for the copied object to create a symlink
-			err = os.Symlink(symlinkData.String(), o.path)
-		}
-
-		// only continue if symlink creation succeeded
-		if err != nil {
-			return err
-		}
-	}
-
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
 		if removeErr := os.Remove(o.path); removeErr != nil {
@@ -1038,36 +875,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	return o.lstat()
 }
 
-// OpenWriterAt opens with a handle for random access writes
-//
-// Pass in the remote desired and the size if known.
-//
-// It truncates any existing object
-func (f *Fs) OpenWriterAt(remote string, size int64) (fs.WriterAtCloser, error) {
-	// Temporary Object under construction
-	o := f.newObject(remote, "")
-
-	err := o.mkdirAll()
-	if err != nil {
-		return nil, err
-	}
-
-	if o.translatedLink {
-		return nil, errors.New("can't open a symlink for random writing")
-	}
-
-	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-	if err != nil {
-		return nil, err
-	}
-	// Pre-allocate the file for performance reasons
-	err = preAllocate(size, out)
-	if err != nil {
-		fs.Debugf(o, "Failed to pre-allocate: %v", err)
-	}
-	return out, nil
-}
-
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
 	// Don't overwrite the info if we don't need to
@@ -1209,11 +1016,10 @@ func cleanWindowsName(f *Fs, name string) string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs             = &Fs{}
-	_ fs.Purger         = &Fs{}
-	_ fs.PutStreamer    = &Fs{}
-	_ fs.Mover          = &Fs{}
-	_ fs.DirMover       = &Fs{}
-	_ fs.OpenWriterAter = &Fs{}
-	_ fs.Object         = &Object{}
+	_ fs.Fs          = &Fs{}
+	_ fs.Purger      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.Mover       = &Fs{}
+	_ fs.DirMover    = &Fs{}
+	_ fs.Object      = &Object{}
 )

backend/local/local_internal_test.go

@@ -1,16 +1,10 @@
 package local
 
 import (
-	"io/ioutil"
-	"os"
 	"path"
-	"path/filepath"
-	"runtime"
 	"testing"
 	"time"
 
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
 	"github.com/ncw/rclone/lib/file"
@@ -48,9 +42,6 @@ func TestUpdatingCheck(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}
-	defer func() {
-		require.NoError(t, fd.Close())
-	}()
 
 	fi, err := fd.Stat()
 	require.NoError(t, err)
@@ -81,108 +72,3 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 
 }
-
-func TestSymlink(t *testing.T) {
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	f := r.Flocal.(*Fs)
-	dir := f.root
-
-	// Write a file
-	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
-	file1 := r.WriteFile("file.txt", "hello", modTime1)
-
-	// Write a symlink
-	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
-	symlinkPath := filepath.Join(dir, "symlink.txt")
-	require.NoError(t, os.Symlink("file.txt", symlinkPath))
-	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
-
-	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
-	if runtime.GOOS == "windows" {
-		file2.Size = 0 // symlinks are 0 length under Windows
-	}
-
-	// Object viewed as destination
-	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
-
-	// Check with no symlink flags
-	fstest.CheckItems(t, r.Flocal, file1)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-L" mode
-	f.opt.FollowSymlinks = true
-	f.opt.TranslateSymlinks = false
-	f.lstat = os.Stat
-
-	fstest.CheckItems(t, r.Flocal, file1, file2d)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-l" mode
-	f.opt.FollowSymlinks = false
-	f.opt.TranslateSymlinks = true
-	f.lstat = os.Lstat
-
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2)
-	}
-
-	// Create a symlink
-	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
-	if runtime.GOOS == "windows" {
-		file3.Size = 0 // symlinks are 0 length under Windows
-	}
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
-	}
-
-	// Check it got the correct contents
-	symlinkPath = filepath.Join(dir, "symlink2.txt")
-	fi, err := os.Lstat(symlinkPath)
-	require.NoError(t, err)
-	assert.False(t, fi.Mode().IsRegular())
-	linkText, err := os.Readlink(symlinkPath)
-	require.NoError(t, err)
-	assert.Equal(t, "file.txt", linkText)
-
-	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
-	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
-	if runtime.GOOS != "windows" {
-		assert.Equal(t, int64(8), o.Size())
-	}
-
-	// Check that NewObject doesn't see the non suffixed version
-	_, err = r.Flocal.NewObject("symlink2.txt")
-	require.Equal(t, fs.ErrorObjectNotFound, err)
-
-	// Check reading the object
-	in, err := o.Open()
-	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt", string(contents))
-	require.NoError(t, in.Close())
-
-	// Check reading the object with range
-	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
-	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt"[2:5+1], string(contents))
-	require.NoError(t, in.Close())
-}
-
-func TestSymlinkError(t *testing.T) {
-	m := configmap.Simple{
-		"links":      "true",
-		"copy_links": "true",
-	}
-	_, err := NewFs("local", "/", m)
-	assert.Equal(t, errLinksAndCopyLinks, err)
-}

backend/local/preallocate_unix.go

@@ -4,40 +4,16 @@ package local
 
 import (
 	"os"
-	"sync/atomic"
 
-	"github.com/ncw/rclone/fs"
 	"golang.org/x/sys/unix"
 )
 
-var (
-	fallocFlags = [...]uint32{
-		unix.FALLOC_FL_KEEP_SIZE,                             // Default
-		unix.FALLOC_FL_KEEP_SIZE | unix.FALLOC_FL_PUNCH_HOLE, // for ZFS #3066
-	}
-	fallocFlagsIndex int32
-)
-
 // preAllocate the file for performance reasons
 func preAllocate(size int64, out *os.File) error {
 	if size <= 0 {
 		return nil
 	}
-	index := atomic.LoadInt32(&fallocFlagsIndex)
-again:
-	if index >= int32(len(fallocFlags)) {
-		return nil // Fallocate is disabled
-	}
-	flags := fallocFlags[index]
-	err := unix.Fallocate(int(out.Fd()), flags, 0, size)
-	if err == unix.ENOTSUP {
-		// Try the next flags combination
-		index++
-		atomic.StoreInt32(&fallocFlagsIndex, index)
-		fs.Debugf(nil, "preAllocate: got error on fallocate, trying combination %d/%d: %v", index, len(fallocFlags), err)
-		goto again
-
-	}
+	err := unix.Fallocate(int(out.Fd()), unix.FALLOC_FL_KEEP_SIZE, 0, size)
 	// FIXME could be doing something here
 	// if err == unix.ENOSPC {
 	// 	log.Printf("No space")

backend/local/read_device_unix.go

@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
 		fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
 		return devUnset
 	}
-	return uint64(statT.Dev) // nolint: unconvert
+	return uint64(statT.Dev)
 }

backend/mega/mega.go

@@ -98,7 +98,7 @@ type Fs struct {
 	opt        Options      // parsed config options
 	features   *fs.Features // optional features
 	srv        *mega.Mega   // the connection to the server
-	pacer      *fs.Pacer    // pacer for API calls
+	pacer      *pacer.Pacer // pacer for API calls
 	rootNodeMu sync.Mutex   // mutex for _rootNode
 	_rootNode  *mega.Node   // root node - call findRoot to use this
 	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir
@@ -217,7 +217,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   srv,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
@@ -402,35 +402,6 @@ func (f *Fs) clearRoot() {
 	//log.Printf("cleared root directory")
 }
 
-// CleanUp deletes all files currently in trash
-func (f *Fs) CleanUp() (err error) {
-	trash := f.srv.FS.GetTrash()
-	items := []*mega.Node{}
-	_, err = f.list(trash, func(item *mega.Node) bool {
-		items = append(items, item)
-		return false
-	})
-	if err != nil {
-		return errors.Wrap(err, "CleanUp failed to list items in trash")
-	}
-	fs.Infof(f, "Deleting %d items from the trash", len(items))
-	errors := 0
-	// similar to f.deleteNode(trash) but with HardDelete as true
-	for _, item := range items {
-		fs.Debugf(f, "Deleting trash %q", item.GetName())
-		deleteErr := f.pacer.Call(func() (bool, error) {
-			err := f.srv.Delete(item, true)
-			return shouldRetry(err)
-		})
-		if deleteErr != nil {
-			err = deleteErr
-			errors++
-		}
-	}
-	fs.Infof(f, "Deleted %d items from the trash with %d errors", len(items), errors)
-	return err
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -526,7 +497,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, object, leaf and error
+// Returns the dirNode, obect, leaf and error
 //
 // Used to create new objects
 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -552,10 +523,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -876,14 +847,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.deleteNode(srcDirNode)
 		if err != nil {
-			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1105,9 +1076,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
-	if size < 0 {
-		return errors.New("mega backend can't upload a file of unknown length")
-	}
 	//modTime := src.ModTime()
 	remote := o.Remote()
 
@@ -1158,7 +1126,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return errors.Wrap(err, "failed to finish upload")
 	}
 
-	// If the upload succeeded and the original object existed, then delete it
+	// If the upload succeded and the original object existed, then delete it
 	if o.info != nil {
 		err = o.fs.deleteNode(o.info)
 		if err != nil {

backend/onedrive/api/types.go

@@ -25,7 +25,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := e.ErrorInfo.Code
 	if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +35,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Identity represents an identity of an actor. For example, and actor
@@ -295,9 +295,9 @@ func (i *Item) GetID() string {
 	return i.ID
 }
 
-// GetDriveID returns a normalized ParentReference of the item
+// GetDriveID returns a normalized ParentReferance of the item
 func (i *Item) GetDriveID() string {
-	return i.GetParentReference().DriveID
+	return i.GetParentReferance().DriveID
 }
 
 // GetName returns a normalized Name of the item
@@ -398,8 +398,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
 	return i.LastModifiedDateTime
 }
 
-// GetParentReference returns a normalized ParentReference of the item
-func (i *Item) GetParentReference() *ItemReference {
+// GetParentReferance returns a normalized ParentReferance of the item
+func (i *Item) GetParentReferance() *ItemReference {
 	if i.IsRemote() && i.ParentReference == nil {
 		return i.RemoteItem.ParentReference
 	}

backend/onedrive/onedrive.go

@@ -14,8 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ncw/rclone/lib/atexit"
-
 	"github.com/ncw/rclone/backend/onedrive/api"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -229,7 +227,7 @@ that the chunks will be buffered into memory.`,
 			Advanced: true,
 		}, {
 			Name:     "drive_type",
-			Help:     "The type of the drive ( " + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + " )",
+			Help:     "The type of the drive ( personal | business | documentLibrary )",
 			Default:  "",
 			Advanced: true,
 		}, {
@@ -263,7 +261,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the one drive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
@@ -326,24 +324,19 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
+	authRety := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
+		authRety = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
 // if `relPath` == "", it reads the metadata for the item with that ID.
-//
-// We address items using the pattern `drives/driveID/items/itemID:/relativePath`
-// instead of simply using `drives/driveID/root:/itemPath` because it works for
-// "shared with me" folders in OneDrive Personal (See #2536, #2778)
-// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
+	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -385,19 +378,9 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var dirCacheRootIDExists bool
-		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
-		if f.root == "" {
-			// if f.root == "", it means f.root is the absolute root of the drive
-			// and its ID should have been found in NewFs
-			dirCacheFoundRoot = dirCacheRootIDExists
-		} else if _, err := f.dirCache.RootParentID(); err == nil {
-			// if root is in a folder, it must have a parent folder, and
-			// if dirCache has found root in NewFs, the parent folder's ID
-			// should be present.
-			// This RootParentID() check is a fix for #3164 which describes
-			// a possible case where the root is not found.
-			dirCacheFoundRoot = dirCacheRootIDExists
+		var ok bool
+		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
+			dirCacheFoundRoot = true
 		}
 	}
 
@@ -492,7 +475,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		driveID:   opt.DriveID,
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
-		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:     pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -509,11 +492,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// Get rootID
 	rootInfo, _, err := f.readMetaDataForPath("")
-	if err != nil || rootInfo.GetID() == "" {
+	if err != nil || rootInfo.ID == "" {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
 
-	f.dirCache = dircache.New(root, rootInfo.GetID(), f)
+	f.dirCache = dircache.New(root, rootInfo.ID, f)
 
 	// Find the current root
 	err = f.dirCache.FindRoot(false)
@@ -720,7 +703,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			id := info.GetID()
 			f.dirCache.Put(remote, id)
 			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-			d.SetItems(folder.ChildCount)
+			if folder != nil {
+				d.SetItems(folder.ChildCount)
+			}
 			entries = append(entries, d)
 		} else {
 			o, err := f.newObjectWithInfo(remote, info)
@@ -834,6 +819,9 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -1352,12 +1340,12 @@ func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
 		}
 	} else {
 		opts = rest.Opts{
 			Method: "PATCH",
-			Path:   "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
+			Path:   "/root:/" + rest.URLPathEscape(o.srvPath()),
 		}
 	}
 	update := api.SetFileSystemInfo{
@@ -1500,43 +1488,26 @@ func (o *Object) cancelUploadSession(url string) (err error) {
 // uploadMultipart uploads a file using multipart upload
 func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size <= 0 {
-		return nil, errors.New("unknown-sized upload not supported")
+		panic("size passed into uploadMultipart must be > 0")
 	}
 
-	uploadURLChan := make(chan string, 1)
-	gracefulCancel := func() {
-		uploadURL, ok := <-uploadURLChan
-		// Reading from uploadURLChan blocks the atexit process until
-		// we are able to use uploadURL to cancel the upload
-		if !ok { // createUploadSession failed - no need to cancel upload
-			return
-		}
-
-		fs.Debugf(o, "Cancelling multipart upload")
-		cancelErr := o.cancelUploadSession(uploadURL)
-		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
-		}
-	}
-	cancelFuncHandle := atexit.Register(gracefulCancel)
-
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(modTime)
 	if err != nil {
-		close(uploadURLChan)
-		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
-	uploadURLChan <- uploadURL
 
+	// Cancel the session if something went wrong
 	defer func() {
 		if err != nil {
-			fs.Debugf(o, "Error encountered during upload: %v", err)
-			gracefulCancel()
+			fs.Debugf(o, "Cancelling multipart upload: %v", err)
+			cancelErr := o.cancelUploadSession(uploadURL)
+			if cancelErr != nil {
+				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
+			}
 		}
-		atexit.Unregister(cancelFuncHandle)
 	}()
 
 	// Upload the chunks
@@ -1564,7 +1535,7 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
 // This function will set modtime after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
-		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
+		panic("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -1589,6 +1560,10 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 		}
 	}
 
+	if size == 0 {
+		opts.Body = nil
+	}
+
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1631,7 +1606,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	} else if size == 0 {
 		info, err = o.uploadSinglepart(in, size, modTime)
 	} else {
-		return errors.New("unknown-sized upload not supported")
+		panic("src file size must be >= 0")
 	}
 	if err != nil {
 		return err
@@ -1697,21 +1672,6 @@ func getRelativePathInsideBase(base, target string) (string, bool) {
 	return "", false
 }
 
-// Adds a ":" at the end of `remotePath` in a proper manner.
-// If `remotePath` already ends with "/", change it to ":/"
-// If `remotePath` is "", return "".
-// A workaround for #2720 and #3039
-func withTrailingColon(remotePath string) string {
-	if remotePath == "" {
-		return ""
-	}
-
-	if strings.HasSuffix(remotePath, "/") {
-		return remotePath[:len(remotePath)-1] + ":/"
-	}
-	return remotePath + ":"
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -65,7 +65,7 @@ type Fs struct {
 	opt      Options            // parsed options
 	features *fs.Features       // optional features
 	srv      *rest.Client       // the connection to the server
-	pacer    *fs.Pacer          // To pace and retry the API calls
+	pacer    *pacer.Pacer       // To pace and retry the API calls
 	session  UserSessionInfo    // contains the session data
 	dirCache *dircache.DirCache // Map of directory path to directory id
 }
@@ -119,7 +119,7 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -144,7 +144,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 
 	f.dirCache = dircache.New(root, "0", f)
@@ -287,6 +287,9 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -782,7 +785,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, folder.FolderID)
-		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
+		d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
 		entries = append(entries, d)
 	}

backend/opendrive/types.go

@@ -13,7 +13,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }

backend/pcloud/api/types.go

@@ -41,7 +41,7 @@ type Error struct {
 	ErrorString string `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
 }
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
 	return e
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Item describes a folder or a file as returned by Get Folder Items and others
@@ -161,6 +161,7 @@ type UserInfo struct {
 	PublicLinkQuota       int64  `json:"publiclinkquota"`
 	Email                 string `json:"email"`
 	UserID                int    `json:"userid"`
+	Result                int    `json:"result"`
 	Quota                 int64  `json:"quota"`
 	TrashRevretentionDays int    `json:"trashrevretentiondays"`
 	Premium               bool   `json:"premium"`

backend/pcloud/pcloud.go

@@ -95,7 +95,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
@@ -254,7 +254,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         false,
@@ -385,7 +385,7 @@ func fileIDtoNumber(fileID string) string {
 	if len(fileID) > 0 && fileID[0] == 'f' {
 		return fileID[1:]
 	}
-	fs.Debugf(nil, "Invalid file id %q", fileID)
+	fs.Debugf(nil, "Invalid filee id %q", fileID)
 	return fileID
 }
 

backend/qingstor/qingstor.go

@@ -101,13 +101,10 @@ enough memory, then increasing this will speed up the transfers.`,
 This is the number of chunks of the same file that are uploaded
 concurrently.
 
-NB if you set this to > 1 then the checksums of multpart uploads
-become corrupted (the uploads themselves are not corrupted though).
-
 If you are uploading small numbers of large file over high speed link
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  1,
+			Default:  4,
 			Advanced: true,
 		}},
 	})
@@ -449,7 +446,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	_, err = bucketInit.PutObject(key, &req)
 	if err != nil {
-		fs.Debugf(f, "Copy Failed, API Error: %v", err)
+		fs.Debugf(f, "Copied Faild, API Error: %v", err)
 		return nil, err
 	}
 	return f.NewObject(remote)
@@ -756,7 +753,7 @@ func (f *Fs) Mkdir(dir string) error {
 		}
 		switch *statistics.Status {
 		case "deleted":
-			fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
+			fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
 			time.Sleep(time.Second * 1)
 			retries++
 			continue
@@ -875,7 +872,7 @@ func (o *Object) readMetaData() (err error) {
 	fs.Debugf(o, "Read metadata of key: %s", key)
 	resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
 	if err != nil {
-		fs.Debugf(o, "Read metadata failed, API Error: %v", err)
+		fs.Debugf(o, "Read metadata faild, API Error: %v", err)
 		if e, ok := err.(*qsErr.QingStorError); ok {
 			if e.StatusCode == http.StatusNotFound {
 				return fs.ErrorObjectNotFound

backend/qingstor/upload.go

@@ -143,7 +143,7 @@ func (u *uploader) init() {
 
 		// Try to adjust partSize if it is too small and account for
 		// integer division truncation.
-		if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
+		if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
 			// Add one to the part size to account for remainders
 			// during the size calculation. e.g odd number of bytes.
 			u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -163,7 +163,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 
 	_, err := bucketInit.PutObject(u.cfg.key, &req)
 	if err == nil {
-		fs.Debugf(u, "Upload single object finished")
+		fs.Debugf(u, "Upload single objcet finished")
 	}
 	return err
 }

backend/s3/s3.go

@@ -131,9 +131,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -237,10 +234,10 @@ func init() {
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.softlayer.net",
-				Help:  "Great Britain Endpoint",
+				Help:  "Great Britan Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
-				Help:  "Great Britain Private Endpoint",
+				Help:  "Great Britan Private Endpoint",
 			}, {
 				Value: "s3.ap-geo.objectstorage.softlayer.net",
 				Help:  "APAC Cross Regional Endpoint",
@@ -346,7 +343,7 @@ func init() {
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
 			Provider: "!AWS,IBMCOS,Alibaba",
 			Examples: []fs.OptionExample{{
-				Value:    "objects-us-east-1.dream.io",
+				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
 			}, {
@@ -369,10 +366,6 @@ func init() {
 				Value:    "s3.us-west-1.wasabisys.com",
 				Help:     "Wasabi US West endpoint",
 				Provider: "Wasabi",
-			}, {
-				Value:    "s3.eu-central-1.wasabisys.com",
-				Help:     "Wasabi EU Central endpoint",
-				Provider: "Wasabi",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -399,9 +392,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -454,7 +444,7 @@ func init() {
 				Help:  "US East Region Flex",
 			}, {
 				Value: "us-south-standard",
-				Help:  "US South Region Standard",
+				Help:  "US Sout hRegion Standard",
 			}, {
 				Value: "us-south-vault",
 				Help:  "US South Region Vault",
@@ -478,16 +468,16 @@ func init() {
 				Help:  "EU Cross Region Flex",
 			}, {
 				Value: "eu-gb-standard",
-				Help:  "Great Britain Standard",
+				Help:  "Great Britan Standard",
 			}, {
 				Value: "eu-gb-vault",
-				Help:  "Great Britain Vault",
+				Help:  "Great Britan Vault",
 			}, {
 				Value: "eu-gb-cold",
-				Help:  "Great Britain Cold",
+				Help:  "Great Britan Cold",
 			}, {
 				Value: "eu-gb-flex",
-				Help:  "Great Britain Flex",
+				Help:  "Great Britan Flex",
 			}, {
 				Value: "ap-standard",
 				Help:  "APAC Standard",
@@ -649,9 +639,6 @@ isn't set then "acl" is used instead.`,
 			}, {
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
-			}, {
-				Value: "DEEP_ARCHIVE",
-				Help:  "Glacier Deep Archive storage class",
 			}},
 		}, {
 			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
@@ -736,14 +723,6 @@ If it is set then rclone will use v2 authentication.
 Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:     "use_accelerate_endpoint",
-			Provider: "AWS",
-			Help: `If true use the AWS S3 accelerated endpoint.
-
-See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
-			Default:  false,
-			Advanced: true,
 		}},
 	})
 }
@@ -764,26 +743,25 @@ const (
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider              string        `config:"provider"`
-	EnvAuth               bool          `config:"env_auth"`
-	AccessKeyID           string        `config:"access_key_id"`
-	SecretAccessKey       string        `config:"secret_access_key"`
-	Region                string        `config:"region"`
-	Endpoint              string        `config:"endpoint"`
-	LocationConstraint    string        `config:"location_constraint"`
-	ACL                   string        `config:"acl"`
-	BucketACL             string        `config:"bucket_acl"`
-	ServerSideEncryption  string        `config:"server_side_encryption"`
-	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
-	StorageClass          string        `config:"storage_class"`
-	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum       bool          `config:"disable_checksum"`
-	SessionToken          string        `config:"session_token"`
-	UploadConcurrency     int           `config:"upload_concurrency"`
-	ForcePathStyle        bool          `config:"force_path_style"`
-	V2Auth                bool          `config:"v2_auth"`
-	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
+	Provider             string        `config:"provider"`
+	EnvAuth              bool          `config:"env_auth"`
+	AccessKeyID          string        `config:"access_key_id"`
+	SecretAccessKey      string        `config:"secret_access_key"`
+	Region               string        `config:"region"`
+	Endpoint             string        `config:"endpoint"`
+	LocationConstraint   string        `config:"location_constraint"`
+	ACL                  string        `config:"acl"`
+	BucketACL            string        `config:"bucket_acl"`
+	ServerSideEncryption string        `config:"server_side_encryption"`
+	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
+	StorageClass         string        `config:"storage_class"`
+	UploadCutoff         fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
+	DisableChecksum      bool          `config:"disable_checksum"`
+	SessionToken         string        `config:"session_token"`
+	UploadConcurrency    int           `config:"upload_concurrency"`
+	ForcePathStyle       bool          `config:"force_path_style"`
+	V2Auth               bool          `config:"v2_auth"`
 }
 
 // Fs represents a remote s3 server
@@ -798,7 +776,7 @@ type Fs struct {
 	bucketOKMu    sync.Mutex       // mutex to protect bucket OK
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
-	pacer         *fs.Pacer        // To pace the API calls
+	pacer         *pacer.Pacer     // To pace the API calls
 	srv           *http.Client     // a plain http client
 }
 
@@ -855,24 +833,15 @@ var retryErrorCodes = []int{
 //S3 is pretty resilient, and the built in retry handling is probably sufficient
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
-func (f *Fs) shouldRetry(err error) (bool, error) {
+func shouldRetry(err error) (bool, error) {
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
-		// Simple case, check the original embedded error in case it's generically retryable
+		// Simple case, check the original embedded error in case it's generically retriable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
 		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
-			// 301 if wrong region for bucket
-			if reqErr.StatusCode() == http.StatusMovedPermanently {
-				urfbErr := f.updateRegionForBucket()
-				if urfbErr != nil {
-					fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
-					return false, err
-				}
-				return true, err
-			}
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
 					return true, err
@@ -957,22 +926,16 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
+		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
+		WithEndpoint(opt.Endpoint).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
-		WithS3ForcePathStyle(opt.ForcePathStyle).
-		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
-	if opt.Region != "" {
-		awsConfig.WithRegion(opt.Region)
-	}
-	if opt.Endpoint != "" {
-		awsConfig.WithEndpoint(opt.Endpoint)
-	}
-
+		WithS3ForcePathStyle(opt.ForcePathStyle)
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
 	awsSessionOpts := session.Options{
 		Config: *awsConfig,
@@ -1072,7 +1035,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		c:      c,
 		bucket: bucket,
 		ses:    ses,
-		pacer:  fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
 		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1089,7 +1052,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.c.HeadObject(&req)
-			return f.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err == nil {
 			f.root = path.Dir(directory)
@@ -1139,51 +1102,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// Gets the bucket location
-func (f *Fs) getBucketLocation() (string, error) {
-	req := s3.GetBucketLocationInput{
-		Bucket: &f.bucket,
-	}
-	var resp *s3.GetBucketLocationOutput
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(err)
-	})
-	if err != nil {
-		return "", err
-	}
-	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
-}
-
-// Updates the region for the bucket by reading the region from the
-// bucket then updating the session.
-func (f *Fs) updateRegionForBucket() error {
-	region, err := f.getBucketLocation()
-	if err != nil {
-		return errors.Wrap(err, "reading bucket location failed")
-	}
-	if aws.StringValue(f.c.Config.Endpoint) != "" {
-		return errors.Errorf("can't set region to %q as endpoint is set", region)
-	}
-	if aws.StringValue(f.c.Config.Region) == region {
-		return errors.Errorf("region is already %q - not updating", region)
-	}
-
-	// Make a new session with the new region
-	oldRegion := f.opt.Region
-	f.opt.Region = region
-	c, ses, err := s3Connection(&f.opt)
-	if err != nil {
-		return errors.Wrap(err, "creating new session failed")
-	}
-	f.c = c
-	f.ses = ses
-
-	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
-	return nil
-}
-
 // listFn is called from list to handle an object.
 type listFn func(remote string, object *s3.Object, isDirectory bool) error
 
@@ -1216,7 +1134,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListObjects(&req)
-			return f.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1345,7 +1263,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	var resp *s3.ListBucketsOutput
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.ListBuckets(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1433,7 +1351,7 @@ func (f *Fs) dirExists() (bool, error) {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.HeadBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err == nil {
 		return true, nil
@@ -1473,7 +1391,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CreateBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err, ok := err.(awserr.Error); ok {
 		if err.Code() == "BucketAlreadyOwnedByYou" {
@@ -1502,7 +1420,7 @@ func (f *Fs) Rmdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.DeleteBucket(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err == nil {
 		f.bucketOK = false
@@ -1563,7 +1481,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.c.CopyObject(&req)
-		return f.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1645,7 +1563,7 @@ func (o *Object) readMetaData() (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1736,15 +1654,12 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	if o.fs.opt.SSEKMSKeyID != "" {
 		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
 	}
-	if o.fs.opt.StorageClass == "GLACIER" || o.fs.opt.StorageClass == "DEEP_ARCHIVE" {
-		return fs.ErrorCantSetModTime
-	}
 	if o.fs.opt.StorageClass != "" {
 		req.StorageClass = &o.fs.opt.StorageClass
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.CopyObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	return err
 }
@@ -1776,7 +1691,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.GetObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
@@ -1867,7 +1782,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		}
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			_, err = uploader.Upload(&req)
-			return o.fs.shouldRetry(err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return err
@@ -1923,11 +1838,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			resp, err := o.fs.srv.Do(httpReq)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return shouldRetry(err)
 			}
 			body, err := rest.ReadBody(resp)
 			if err != nil {
-				return o.fs.shouldRetry(err)
+				return shouldRetry(err)
 			}
 			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
 				return false, nil
@@ -1955,7 +1870,7 @@ func (o *Object) Remove() error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.DeleteObject(&req)
-		return o.fs.shouldRetry(err)
+		return shouldRetry(err)
 	})
 	return err
 }

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 
@@ -14,7 +14,6 @@ import (
 	"os/user"
 	"path"
 	"regexp"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -26,7 +25,6 @@ import (
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/env"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
@@ -188,10 +186,10 @@ func readCurrentUser() (userName string) {
 	return os.Getenv("LOGNAME")
 }
 
-// dial starts a client connection to the given SSH server. It is a
+// Dial starts a client connection to the given SSH server. It is a
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
-func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
+func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
 	dialer := fshttp.NewDialer(fs.Config)
 	conn, err := dialer.Dial(network, addr)
 	if err != nil {
@@ -201,7 +199,6 @@ func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Clien
 	if err != nil {
 		return nil, err
 	}
-	fs.Debugf(f, "New connection %s->%s to %q", c.LocalAddr(), c.RemoteAddr(), c.ServerVersion())
 	return ssh.NewClient(c, chans, reqs), nil
 }
 
@@ -247,7 +244,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	c = &conn{
 		err: make(chan error, 1),
 	}
-	c.sshClient, err = f.dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
+	c.sshClient, err = Dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't connect SSH")
 	}
@@ -318,6 +315,18 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
+// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
+// variables afterwards.
+func shellExpand(s string) string {
+	if s != "" {
+		if s[0] == '~' {
+			s = "${HOME}" + s[1:]
+		}
+		s = os.ExpandEnv(s)
+	}
+	return s
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -338,7 +347,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		Auth:            []ssh.AuthMethod{},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 		Timeout:         fs.Config.ConnectTimeout,
-		ClientVersion:   "SSH-2.0-" + fs.Config.UserAgent,
 	}
 
 	if opt.UseInsecureCipher {
@@ -346,7 +354,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
-	keyFile := env.ShellExpand(opt.KeyFile)
+	keyFile := shellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
 	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
@@ -419,12 +427,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
-	return NewFsWithConnection(name, root, opt, sshConfig)
-}
-
-// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
-// the host specified in the ssh.ClientConfig
-func NewFsWithConnection(name string, root string, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
 		name:      name,
 		root:      root,
@@ -563,13 +565,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
 		if !info.Mode().IsRegular() {
-			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
-				if !os.IsNotExist(err) {
-					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
-				}
-				info = oldInfo
+				return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
 			}
 		}
 		if info.IsDir() {
@@ -805,49 +803,6 @@ func (f *Fs) Hashes() hash.Set {
 	return set
 }
 
-// About gets usage stats
-func (f *Fs) About() (*fs.Usage, error) {
-	c, err := f.getSftpConnection()
-	if err != nil {
-		return nil, errors.Wrap(err, "About get SFTP connection")
-	}
-	session, err := c.sshClient.NewSession()
-	f.putSftpConnection(&c, err)
-	if err != nil {
-		return nil, errors.Wrap(err, "About put SFTP connection")
-	}
-
-	var stdout, stderr bytes.Buffer
-	session.Stdout = &stdout
-	session.Stderr = &stderr
-	escapedPath := shellEscape(f.root)
-	if f.opt.PathOverride != "" {
-		escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
-	}
-	if len(escapedPath) == 0 {
-		escapedPath = "/"
-	}
-	err = session.Run("df -k " + escapedPath)
-	if err != nil {
-		_ = session.Close()
-		return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.")
-	}
-	_ = session.Close()
-
-	usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes())
-	usage := &fs.Usage{}
-	if usageTotal >= 0 {
-		usage.Total = fs.NewUsageValue(usageTotal)
-	}
-	if usageUsed >= 0 {
-		usage.Used = fs.NewUsageValue(usageUsed)
-	}
-	if usageAvail >= 0 {
-		usage.Free = fs.NewUsageValue(usageAvail)
-	}
-	return usage, nil
-}
-
 // Fs is the filesystem this remote sftp file object is located within
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -938,35 +893,6 @@ func parseHash(bytes []byte) string {
 	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
 }
 
-// Parses the byte array output from the SSH session
-// returned by an invocation of df into
-// the disk size, used space, and avaliable space on the disk, in that order.
-// Only works when `df` has output info on only one disk
-func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
-	spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
-	lines := strings.Split(string(bytes), "\n")
-	if len(lines) < 2 {
-		return
-	}
-	split := strings.Fields(lines[1])
-	if len(split) < 6 {
-		return
-	}
-	spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
-	if err != nil {
-		spaceTotal = -1
-	}
-	spaceUsed, err = strconv.ParseInt(split[2], 10, 64)
-	if err != nil {
-		spaceUsed = -1
-	}
-	spaceAvail, err = strconv.ParseInt(split[3], 10, 64)
-	if err != nil {
-		spaceAvail = -1
-	}
-	return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
-}
-
 // Size returns the size in bytes of the remote sftp file
 func (o *Object) Size() int64 {
 	return o.size

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 
@@ -35,17 +35,3 @@ func TestParseHash(t *testing.T) {
 		assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
 	}
 }
-
-func TestParseUsage(t *testing.T) {
-	for i, test := range []struct {
-		sshOutput string
-		usage     [3]int64
-	}{
-		{"Filesystem     1K-blocks     Used Available Use% Mounted on\n/dev/root       91283092 81111888  10154820  89% /", [3]int64{93473886208, 83058573312, 10398535680}},
-		{"Filesystem     1K-blocks  Used Available Use% Mounted on\ntmpfs             818256  1636    816620   1% /run", [3]int64{837894144, 1675264, 836218880}},
-		{"Filesystem   1024-blocks     Used Available Capacity iused      ifree %iused  Mounted on\n/dev/disk0s2   244277768 94454848 149566920    39%  997820 4293969459    0%   /", [3]int64{250140434432, 96721764352, 153156526080}},
-	} {
-		gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput))
-		assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
-	}
-}

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9
+// +build plan9 !go1.9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,4 +1,4 @@
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 

backend/sftp/stringlock_test.go

@@ -1,4 +1,4 @@
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 

backend/swift/auth.go

@@ -2,7 +2,6 @@ package swift
 
 import (
 	"net/http"
-	"time"
 
 	"github.com/ncw/swift"
 )
@@ -66,14 +65,6 @@ func (a *auth) Token() string {
 	return a.parentAuth.Token()
 }
 
-// Expires returns the time the token expires if known or Zero if not.
-func (a *auth) Expires() (t time.Time) {
-	if do, ok := a.parentAuth.(swift.Expireser); ok {
-		t = do.Expires()
-	}
-	return t
-}
-
 // The CDN url if available
 func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
@@ -83,7 +74,4 @@ func (a *auth) CdnUrl() string { // nolint
 }
 
 // Check the interfaces are satisfied
-var (
-	_ swift.Authenticator = (*auth)(nil)
-	_ swift.Expireser     = (*auth)(nil)
-)
+var _ swift.Authenticator = (*auth)(nil)

backend/swift/swift.go

@@ -130,15 +130,6 @@ func init() {
 		}, {
 			Name: "auth_token",
 			Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
-		}, {
-			Name: "application_credential_id",
-			Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)",
-		}, {
-			Name: "application_credential_name",
-			Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)",
-		}, {
-			Name: "application_credential_secret",
-			Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)",
 		}, {
 			Name:    "auth_version",
 			Help:    "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
@@ -182,26 +173,23 @@ provider.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth                     bool          `config:"env_auth"`
-	User                        string        `config:"user"`
-	Key                         string        `config:"key"`
-	Auth                        string        `config:"auth"`
-	UserID                      string        `config:"user_id"`
-	Domain                      string        `config:"domain"`
-	Tenant                      string        `config:"tenant"`
-	TenantID                    string        `config:"tenant_id"`
-	TenantDomain                string        `config:"tenant_domain"`
-	Region                      string        `config:"region"`
-	StorageURL                  string        `config:"storage_url"`
-	AuthToken                   string        `config:"auth_token"`
-	AuthVersion                 int           `config:"auth_version"`
-	ApplicationCredentialID     string        `config:"application_credential_id"`
-	ApplicationCredentialName   string        `config:"application_credential_name"`
-	ApplicationCredentialSecret string        `config:"application_credential_secret"`
-	StoragePolicy               string        `config:"storage_policy"`
-	EndpointType                string        `config:"endpoint_type"`
-	ChunkSize                   fs.SizeSuffix `config:"chunk_size"`
-	NoChunk                     bool          `config:"no_chunk"`
+	EnvAuth       bool          `config:"env_auth"`
+	User          string        `config:"user"`
+	Key           string        `config:"key"`
+	Auth          string        `config:"auth"`
+	UserID        string        `config:"user_id"`
+	Domain        string        `config:"domain"`
+	Tenant        string        `config:"tenant"`
+	TenantID      string        `config:"tenant_id"`
+	TenantDomain  string        `config:"tenant_domain"`
+	Region        string        `config:"region"`
+	StorageURL    string        `config:"storage_url"`
+	AuthToken     string        `config:"auth_token"`
+	AuthVersion   int           `config:"auth_version"`
+	StoragePolicy string        `config:"storage_policy"`
+	EndpointType  string        `config:"endpoint_type"`
+	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
+	NoChunk       bool          `config:"no_chunk"`
 }
 
 // Fs represents a remote swift server
@@ -216,7 +204,7 @@ type Fs struct {
 	containerOK       bool              // true if we have created the container
 	segmentsContainer string            // container to store the segments (if any) in
 	noCheckContainer  bool              // don't check the container before creating it
-	pacer             *fs.Pacer         // To pace the API calls
+	pacer             *pacer.Pacer      // To pace the API calls
 }
 
 // Object describes a swift object
@@ -286,31 +274,6 @@ func shouldRetry(err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
-// shouldRetryHeaders returns a boolean as to whether this err
-// deserves to be retried.  It reads the headers passed in looking for
-// `Retry-After`. It returns the err as a convenience
-func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
-	if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
-		if value := headers["Retry-After"]; value != "" {
-			retryAfter, parseErr := strconv.Atoi(value)
-			if parseErr != nil {
-				fs.Errorf(nil, "Failed to parse Retry-After: %q: %v", value, parseErr)
-			} else {
-				duration := time.Second * time.Duration(retryAfter)
-				if duration <= 60*time.Second {
-					// Do a short sleep immediately
-					fs.Debugf(nil, "Sleeping for %v to obey Retry-After", duration)
-					time.Sleep(duration)
-					return true, err
-				}
-				// Delay a long sleep for a retry
-				return false, fserrors.NewErrorRetryAfter(duration)
-			}
-		}
-	}
-	return shouldRetry(err)
-}
-
 // Pattern to match a swift path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -330,25 +293,22 @@ func parsePath(path string) (container, directory string, err error) {
 func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	c := &swift.Connection{
 		// Keep these in the same order as the Config for ease of checking
-		UserName:                    opt.User,
-		ApiKey:                      opt.Key,
-		AuthUrl:                     opt.Auth,
-		UserId:                      opt.UserID,
-		Domain:                      opt.Domain,
-		Tenant:                      opt.Tenant,
-		TenantId:                    opt.TenantID,
-		TenantDomain:                opt.TenantDomain,
-		Region:                      opt.Region,
-		StorageUrl:                  opt.StorageURL,
-		AuthToken:                   opt.AuthToken,
-		AuthVersion:                 opt.AuthVersion,
-		ApplicationCredentialId:     opt.ApplicationCredentialID,
-		ApplicationCredentialName:   opt.ApplicationCredentialName,
-		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
-		EndpointType:                swift.EndpointType(opt.EndpointType),
-		ConnectTimeout:              10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:                     10 * fs.Config.Timeout,        // Use the timeouts in the transport
-		Transport:                   fshttp.NewTransport(fs.Config),
+		UserName:       opt.User,
+		ApiKey:         opt.Key,
+		AuthUrl:        opt.Auth,
+		UserId:         opt.UserID,
+		Domain:         opt.Domain,
+		Tenant:         opt.Tenant,
+		TenantId:       opt.TenantID,
+		TenantDomain:   opt.TenantDomain,
+		Region:         opt.Region,
+		StorageUrl:     opt.StorageURL,
+		AuthToken:      opt.AuthToken,
+		AuthVersion:    opt.AuthVersion,
+		EndpointType:   swift.EndpointType(opt.EndpointType),
+		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:      fshttp.NewTransport(fs.Config),
 	}
 	if opt.EnvAuth {
 		err := c.ApplyEnvironment()
@@ -358,13 +318,11 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	}
 	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
-		if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
-			if c.UserName == "" && c.UserId == "" {
-				return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
-			}
-			if c.ApiKey == "" {
-				return nil, errors.New("key not found")
-			}
+		if c.UserName == "" && c.UserId == "" {
+			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
+		}
+		if c.ApiKey == "" {
+			return nil, errors.New("key not found")
 		}
 		if c.AuthUrl == "" {
 			return nil, errors.New("auth not found")
@@ -426,7 +384,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		segmentsContainer: container + "_segments",
 		root:              directory,
 		noCheckContainer:  noCheckContainer,
-		pacer:             fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+		pacer:             pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -438,9 +396,8 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		// Check to see if the object exists - ignoring directory markers
 		var info swift.Object
 		err = f.pacer.Call(func() (bool, error) {
-			var rxHeaders swift.Headers
-			info, rxHeaders, err = f.c.Object(container, directory)
-			return shouldRetryHeaders(rxHeaders, err)
+			info, _, err = f.c.Object(container, directory)
+			return shouldRetry(err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			f.root = path.Dir(directory)
@@ -456,7 +413,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 	return f, nil
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -749,9 +706,8 @@ func (f *Fs) Mkdir(dir string) error {
 	var err error = swift.ContainerNotFound
 	if !f.noCheckContainer {
 		err = f.pacer.Call(func() (bool, error) {
-			var rxHeaders swift.Headers
-			_, rxHeaders, err = f.c.Container(f.container)
-			return shouldRetryHeaders(rxHeaders, err)
+			_, _, err = f.c.Container(f.container)
+			return shouldRetry(err)
 		})
 	}
 	if err == swift.ContainerNotFound {
@@ -843,9 +799,8 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	srcFs := srcObj.fs
 	err = f.pacer.Call(func() (bool, error) {
-		var rxHeaders swift.Headers
-		rxHeaders, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
-		return shouldRetryHeaders(rxHeaders, err)
+		_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -955,7 +910,7 @@ func (o *Object) readMetaData() (err error) {
 	var h swift.Headers
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, h, err = o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
-		return shouldRetryHeaders(h, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
@@ -1030,9 +985,8 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	headers := fs.OpenOptionHeaders(options)
 	_, isRanging := headers["Range"]
 	err = o.fs.pacer.Call(func() (bool, error) {
-		var rxHeaders swift.Headers
-		in, rxHeaders, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
-		return shouldRetryHeaders(rxHeaders, err)
+		in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+		return shouldRetry(err)
 	})
 	return
 }
@@ -1103,9 +1057,8 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	// Create the segmentsContainer if it doesn't exist
 	var err error
 	err = o.fs.pacer.Call(func() (bool, error) {
-		var rxHeaders swift.Headers
-		_, rxHeaders, err = o.fs.c.Container(o.fs.segmentsContainer)
-		return shouldRetryHeaders(rxHeaders, err)
+		_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
+		return shouldRetry(err)
 	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
@@ -1145,9 +1098,8 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 		segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
 		fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			var rxHeaders swift.Headers
-			rxHeaders, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
-			return shouldRetryHeaders(rxHeaders, err)
+			_, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return "", err
@@ -1160,9 +1112,8 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	emptyReader := bytes.NewReader(nil)
 	manifestName := o.fs.root + o.remote
 	err = o.fs.pacer.Call(func() (bool, error) {
-		var rxHeaders swift.Headers
-		rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
-		return shouldRetryHeaders(rxHeaders, err)
+		_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+		return shouldRetry(err)
 	})
 	return uniquePrefix + "/", err
 }
@@ -1206,7 +1157,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		var rxHeaders swift.Headers
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
-			return shouldRetryHeaders(rxHeaders, err)
+			return shouldRetry(err)
 		})
 		if err != nil {
 			return err

backend/swift/swift_internal_test.go

@@ -1,13 +1,6 @@
 package swift
 
-import (
-	"testing"
-	"time"
-
-	"github.com/ncw/rclone/fs/fserrors"
-	"github.com/ncw/swift"
-	"github.com/stretchr/testify/assert"
-)
+import "testing"
 
 func TestInternalUrlEncode(t *testing.T) {
 	for _, test := range []struct {
@@ -30,37 +23,3 @@ func TestInternalUrlEncode(t *testing.T) {
 		}
 	}
 }
-
-func TestInternalShouldRetryHeaders(t *testing.T) {
-	headers := swift.Headers{
-		"Content-Length": "64",
-		"Content-Type":   "text/html; charset=UTF-8",
-		"Date":           "Mon: 18 Mar 2019 12:11:23 GMT",
-		"Retry-After":    "1",
-	}
-	err := &swift.Error{
-		StatusCode: 429,
-		Text:       "Too Many Requests",
-	}
-
-	// Short sleep should just do the sleep
-	start := time.Now()
-	retry, gotErr := shouldRetryHeaders(headers, err)
-	dt := time.Since(start)
-	assert.True(t, retry)
-	assert.Equal(t, err, gotErr)
-	assert.True(t, dt > time.Second/2)
-
-	// Long sleep should return RetryError
-	headers["Retry-After"] = "3600"
-	start = time.Now()
-	retry, gotErr = shouldRetryHeaders(headers, err)
-	dt = time.Since(start)
-	assert.True(t, dt < time.Second)
-	assert.False(t, retry)
-	assert.Equal(t, true, fserrors.IsRetryAfterError(gotErr))
-	after := gotErr.(fserrors.RetryAfter).RetryAfter()
-	dt = after.Sub(start)
-	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
-
-}

backend/union/union.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/hash"
@@ -178,8 +177,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // At least one value will be written to the channel,
 // specifying the initial value and updated values might
 // follow. A 0 Duration should pause the polling.
-// The ChangeNotify implementation must empty the channel
-// regularly. When the channel gets closed, the implementation
+// The ChangeNotify implemantion must empty the channel
+// regulary. When the channel gets closed, the implemantion
 // should stop polling and release resources.
 func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
 	var remoteChans []chan time.Duration
@@ -343,7 +342,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if configName != "local" {
 			rootString = configName + ":" + rootString
 		}
-		myFs, err := cache.Get(rootString)
+		myFs, err := fs.NewFs(rootString)
 		if err != nil {
 			if err == fs.ErrorIsFile {
 				return myFs, err
@@ -377,11 +376,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	features = features.Mask(f.wr) // mask the features just on the writable fs
 
-	// Really need the union of all remotes for these, so
-	// re-instate and calculate separately.
-	features.ChangeNotify = f.ChangeNotify
-	features.DirCacheFlush = f.DirCacheFlush
-
 	// FIXME maybe should be masking the bools here?
 
 	// Clear ChangeNotify and DirCacheFlush if all are nil

backend/webdav/api/types.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/hash"
 )
 
 const (
@@ -66,13 +65,11 @@ type Response struct {
 // Note that status collects all the status values for which we just
 // check the first is OK.
 type Prop struct {
-	Status       []string  `xml:"DAV: status"`
-	Name         string    `xml:"DAV: prop>displayname,omitempty"`
-	Type         *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
-	IsCollection *string   `xml:"DAV: prop>iscollection,omitempty"` // this is a Microsoft extension see #2716
-	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
-	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
-	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
+	Status   []string  `xml:"DAV: status"`
+	Name     string    `xml:"DAV: prop>displayname,omitempty"`
+	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -98,33 +95,13 @@ func (p *Prop) StatusOK() bool {
 	return false
 }
 
-// Hashes returns a map of all checksums - may be nil
-func (p *Prop) Hashes() (hashes map[hash.Type]string) {
-	if len(p.Checksums) == 0 {
-		return nil
-	}
-	hashes = make(map[hash.Type]string)
-	for _, checksums := range p.Checksums {
-		checksums = strings.ToLower(checksums)
-		for _, checksum := range strings.Split(checksums, " ") {
-			switch {
-			case strings.HasPrefix(checksum, "sha1:"):
-				hashes[hash.SHA1] = checksum[5:]
-			case strings.HasPrefix(checksum, "md5:"):
-				hashes[hash.MD5] = checksum[4:]
-			}
-		}
-	}
-	return hashes
-}
-
 // PropValue is a tagged name and value
 type PropValue struct {
 	XMLName xml.Name `xml:""`
 	Value   string   `xml:",chardata"`
 }
 
-// Error is used to describe webdav errors
+// Error is used to desribe webdav errors
 //
 // <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
 //   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
@@ -137,7 +114,7 @@ type Error struct {
 	StatusCode int
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	var out []string
 	if e.Message != "" {
@@ -210,22 +187,3 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	}
 	return err
 }
-
-// Quota is used to read the bytes used and available
-//
-// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
-//  <d:response>
-//   <d:href>/remote.php/webdav/</d:href>
-//   <d:propstat>
-//    <d:prop>
-//     <d:quota-available-bytes>-3</d:quota-available-bytes>
-//     <d:quota-used-bytes>376461895</d:quota-used-bytes>
-//    </d:prop>
-//    <d:status>HTTP/1.1 200 OK</d:status>
-//   </d:propstat>
-//  </d:response>
-// </d:multistatus>
-type Quota struct {
-	Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
-	Used      int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
-}

backend/webdav/odrvcookie/fetch.go

@@ -102,7 +102,7 @@ func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
 func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
 	spRoot, err := url.Parse(ca.endpoint)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
+		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
 	}
 
 	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
@@ -121,7 +121,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 		Jar: jar,
 	}
 
-	// Send the previously acquired Token as a Post parameter
+	// Send the previously aquired Token as a Post parameter
 	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
 		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
 	}

backend/webdav/odrvcookie/renew.go

@@ -2,10 +2,13 @@ package odrvcookie
 
 import (
 	"time"
+
+	"github.com/ncw/rclone/lib/rest"
 )
 
 // CookieRenew holds information for the renew
 type CookieRenew struct {
+	srv     *rest.Client
 	timer   *time.Ticker
 	renewFn func()
 }

backend/webdav/webdav.go

@@ -2,13 +2,23 @@
 // object storage system.
 package webdav
 
+// Owncloud: Getting Oc-Checksum:
+// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
+// nextcloud?
+
+// docs for file webdav
+// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
+
+// indicates checksums can be set as metadata here
+// https://github.com/nextcloud/server/issues/6129
+// owncloud seems to have checksums as metadata though - can read them
+
 // SetModTime might be possible
 // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
 // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
 // For example the ownCloud WebDAV server does it that way.
 
 import (
-	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -101,12 +111,11 @@ type Fs struct {
 	endpoint           *url.URL      // URL of the host
 	endpointURL        string        // endpoint as a string
 	srv                *rest.Client  // the connection to the one drive server
-	pacer              *fs.Pacer     // pacer for API calls
+	pacer              *pacer.Pacer  // pacer for API calls
 	precision          time.Duration // mod time precision
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
-	hasChecksums       bool          // set if can use owncloud style checksums
 }
 
 // Object describes a webdav object
@@ -118,8 +127,7 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	sha1        string    // SHA-1 of the object content if known
-	md5         string    // MD5 of the object content if known
+	sha1        string    // SHA-1 of the object content
 }
 
 // ------------------------------------------------------------
@@ -146,7 +154,6 @@ func (f *Fs) Features() *fs.Features {
 
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
-	423, // Locked
 	429, // Too Many Requests.
 	500, // Internal Server Error
 	502, // Bad Gateway
@@ -173,18 +180,6 @@ func itemIsDir(item *api.Response) bool {
 		}
 		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
 	}
-	// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
-	// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
-	if t := item.Props.IsCollection; t != nil {
-		switch x := strings.ToLower(*t); x {
-		case "0", "false":
-			return false
-		case "1", "true":
-			return true
-		default:
-			fs.Debugf(nil, "Unknown value %q for IsCollection", x)
-		}
-	}
 	return false
 }
 
@@ -199,9 +194,6 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
 		},
 		NoRedirect: true,
 	}
-	if f.hasChecksums {
-		opts.Body = bytes.NewBuffer(owncloudProps)
-	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -257,7 +249,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// addSlash makes sure s is terminated with a / if non empty
+// addShlash makes sure s is terminated with a / if non empty
 func addSlash(s string) string {
 	if s != "" && !strings.HasSuffix(s, "/") {
 		s += "/"
@@ -319,7 +311,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		endpoint:    u,
 		endpointURL: u.String(),
 		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
-		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		precision:   fs.ModTimeNotSupported,
 	}
 	f.features = (&fs.Features{
@@ -365,11 +357,9 @@ func (f *Fs) setQuirks(vendor string) error {
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -436,22 +426,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// Read the normal props, plus the checksums
-//
-// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
-var owncloudProps = []byte(`<?xml version="1.0"?>
-<d:propfind  xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
- <d:prop>
-  <d:displayname />
-  <d:getlastmodified />
-  <d:getcontentlength />
-  <d:resourcetype />
-  <d:getcontenttype />
-  <oc:checksums />
- </d:prop>
-</d:propfind>
-`)
-
 // list the objects into the function supplied
 //
 // If directories is set it only sends directories
@@ -471,9 +445,6 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
 			"Depth": depth,
 		},
 	}
-	if f.hasChecksums {
-		opts.Body = bytes.NewBuffer(owncloudProps)
-	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -645,18 +616,10 @@ func (f *Fs) _mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	err := f.pacer.Call(func() (bool, error) {
+	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
-	if apiErr, ok := err.(*api.Error); ok {
-		// already exists
-		// owncloud returns 423/StatusLocked if the create is already in progress
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
-			return nil
-		}
-	}
-	return err
 }
 
 // mkdir makes the directory and parents using native paths
@@ -664,7 +627,11 @@ func (f *Fs) mkdir(dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
-		// parent does not exist so create it first then try again
+		// already exists
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
+			return nil
+		}
+		// parent does not exist
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
@@ -880,54 +847,9 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	if f.hasChecksums {
-		return hash.NewHashSet(hash.MD5, hash.SHA1)
-	}
 	return hash.Set(hash.None)
 }
 
-// About gets quota information
-func (f *Fs) About() (*fs.Usage, error) {
-	opts := rest.Opts{
-		Method: "PROPFIND",
-		Path:   "",
-		ExtraHeaders: map[string]string{
-			"Depth": "0",
-		},
-	}
-	opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
-<D:propfind xmlns:D="DAV:">
- <D:prop>
-  <D:quota-available-bytes/>
-  <D:quota-used-bytes/>
- </D:prop>
-</D:propfind>
-`))
-	var q = api.Quota{
-		Available: -1,
-		Used:      -1,
-	}
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &q)
-		return shouldRetry(resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "about call failed")
-	}
-	usage := &fs.Usage{}
-	if q.Available != 0 || q.Used != 0 {
-		if q.Available >= 0 && q.Used >= 0 {
-			usage.Total = fs.NewUsageValue(q.Available + q.Used)
-		}
-		if q.Used >= 0 {
-			usage.Used = fs.NewUsageValue(q.Used)
-		}
-	}
-	return usage, nil
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -948,17 +870,12 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
+// Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(t hash.Type) (string, error) {
-	if o.fs.hasChecksums {
-		switch t {
-		case hash.SHA1:
-			return o.sha1, nil
-		case hash.MD5:
-			return o.md5, nil
-		}
+	if t != hash.SHA1 {
+		return "", hash.ErrUnsupported
 	}
-	return "", hash.ErrUnsupported
+	return o.sha1, nil
 }
 
 // Size returns the size of an object in bytes
@@ -976,11 +893,6 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
-	if o.fs.hasChecksums {
-		hashes := info.Hashes()
-		o.sha1 = hashes[hash.SHA1]
-		o.md5 = hashes[hash.MD5]
-	}
 	return nil
 }
 
@@ -1060,21 +972,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
 		ContentType:   fs.MimeType(src),
 	}
-	if o.fs.useOCMtime || o.fs.hasChecksums {
-		opts.ExtraHeaders = map[string]string{}
-		if o.fs.useOCMtime {
-			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
-		}
-		if o.fs.hasChecksums {
-			// Set an upload checksum - prefer SHA1
-			//
-			// This is used as an upload integrity test. If we set
-			// only SHA1 here, owncloud will calculate the MD5 too.
-			if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
-			} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
-				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
-			}
+	if o.fs.useOCMtime {
+		opts.ExtraHeaders = map[string]string{
+			"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
 		}
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1118,6 +1018,5 @@ var (
 	_ fs.Copier      = (*Fs)(nil)
 	_ fs.Mover       = (*Fs)(nil)
 	_ fs.DirMover    = (*Fs)(nil)
-	_ fs.Abouter     = (*Fs)(nil)
 	_ fs.Object      = (*Object)(nil)
 )

backend/yandex/api/types.go

@@ -56,7 +56,7 @@ type AsyncInfo struct {
 	Templated bool   `json:"templated"`
 }
 
-// AsyncStatus is returned when requesting the status of an async operations. Possible values in-progress, success, failure
+// AsyncStatus is returned when requesting the status of an async operations. Possble values in-progress, success, failure
 type AsyncStatus struct {
 	Status string `json:"status"`
 }

backend/yandex/yandex.go

@@ -93,7 +93,7 @@ type Fs struct {
 	opt      Options      // parsed options
 	features *fs.Features // optional features
 	srv      *rest.Client // the connection to the yandex server
-	pacer    *fs.Pacer    // pacer for API calls
+	pacer    *pacer.Pacer // pacer for API calls
 	diskRoot string       // root path with "disk:/" container name
 }
 
@@ -269,7 +269,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -307,7 +307,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs
 		if err != nil {
 			return nil, errors.Wrap(err, "error parsing time in directory item")
 		}
-		d := fs.NewDir(remote, t).SetSize(object.Size)
+		d := fs.NewDir(remote, t).SetSize(int64(object.Size))
 		return d, nil
 	case "file":
 		o, err := f.newObjectWithInfo(remote, object)
@@ -634,7 +634,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copies or moves directories or files depending on the method parameter
+// copyOrMoves copys or moves directories or files depending on the mthod parameter
 func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -1107,7 +1107,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	//if file uploaded successfully then return metadata
+	//if file uploaded sucessfully then return metadata
 	o.modTime = modTime
 	o.md5sum = ""                   // according to unit tests after put the md5 is empty.
 	o.size = int64(in1.BytesRead()) // better solution o.readMetaData() ?

bin/get-github-release.go

@@ -8,8 +8,6 @@
 package main
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -17,18 +15,13 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
-	"runtime"
 	"strings"
 	"time"
 
-	"github.com/ncw/rclone/lib/rest"
-	"golang.org/x/net/html"
 	"golang.org/x/sys/unix"
 )
 
@@ -37,15 +30,8 @@ var (
 	install = flag.Bool("install", false, "Install the downloaded package using sudo dpkg -i.")
 	extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.")
 	bindir  = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.")
-	useAPI  = flag.Bool("use-api", false, "Use the API for finding the release instead of scraping the page.")
 	// Globals
-	matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
-	osAliases    = map[string][]string{
-		"darwin": []string{"macos", "osx"},
-	}
-	archAliases = map[string][]string{
-		"amd64": []string{"x86_64"},
-	}
+	matchProject = regexp.MustCompile(`^(\w+)/(\w+)$`)
 )
 
 // A github release
@@ -127,41 +113,25 @@ func writable(path string) bool {
 
 // Directory to install releases in by default
 //
-// Find writable directories on $PATH.  Use $GOPATH/bin if that is on
-// the path and writable or use the first writable directory which is
-// in $HOME or failing that the first writable directory.
+// Find writable directories on $PATH.  Use the first writable
+// directory which is in $HOME or failing that the first writable
+// directory.
 //
 // Returns "" if none of the above were found
 func defaultBinDir() string {
 	home := os.Getenv("HOME")
-	var (
-		bin       string
-		homeBin   string
-		goHomeBin string
-		gopath    = os.Getenv("GOPATH")
-	)
+	var binDir string
 	for _, dir := range strings.Split(os.Getenv("PATH"), ":") {
 		if writable(dir) {
 			if strings.HasPrefix(dir, home) {
-				if homeBin != "" {
-					homeBin = dir
-				}
-				if gopath != "" && strings.HasPrefix(dir, gopath) && goHomeBin == "" {
-					goHomeBin = dir
-				}
+				return dir
 			}
-			if bin == "" {
-				bin = dir
+			if binDir != "" {
+				binDir = dir
 			}
 		}
 	}
-	if goHomeBin != "" {
-		return goHomeBin
-	}
-	if homeBin != "" {
-		return homeBin
-	}
-	return bin
+	return binDir
 }
 
 // read the body or an error message
@@ -205,8 +175,7 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	}
 
 	for _, asset := range release.Assets {
-		//log.Printf("Finding %s", asset.Name)
-		if matchName.MatchString(asset.Name) && isOurOsArch(asset.Name) {
+		if matchName.MatchString(asset.Name) {
 			return asset.BrowserDownloadURL, asset.Name
 		}
 	}
@@ -214,73 +183,6 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	return "", ""
 }
 
-// Get an asset URL and name by scraping the downloads page
-//
-// This doesn't use the API so isn't rate limited when not using GITHUB login details
-func getAssetFromReleasesPage(project string, matchName *regexp.Regexp) (assetURL string, assetName string) {
-	baseURL := "https://github.com/" + project + "/releases"
-	log.Printf("Fetching asset info for %q from %q", project, baseURL)
-	base, err := url.Parse(baseURL)
-	if err != nil {
-		log.Fatalf("URL Parse failed: %v", err)
-	}
-	resp, err := http.Get(baseURL)
-	if err != nil {
-		log.Fatalf("Failed to fetch release info %q: %v", baseURL, err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		log.Printf("Error: %s", readBody(resp.Body))
-		log.Fatalf("Bad status %d when fetching %q release info: %s", resp.StatusCode, baseURL, resp.Status)
-	}
-	doc, err := html.Parse(resp.Body)
-	if err != nil {
-		log.Fatalf("Failed to parse web page: %v", err)
-	}
-	var walk func(*html.Node)
-	walk = func(n *html.Node) {
-		if n.Type == html.ElementNode && n.Data == "a" {
-			for _, a := range n.Attr {
-				if a.Key == "href" {
-					if name := path.Base(a.Val); matchName.MatchString(name) && isOurOsArch(name) {
-						if u, err := rest.URLJoin(base, a.Val); err == nil {
-							if assetName == "" {
-								assetName = name
-								assetURL = u.String()
-							}
-						}
-					}
-					break
-				}
-			}
-		}
-		for c := n.FirstChild; c != nil; c = c.NextSibling {
-			walk(c)
-		}
-	}
-	walk(doc)
-	if assetName == "" || assetURL == "" {
-		log.Fatalf("Didn't find URL in page")
-	}
-	return assetURL, assetName
-}
-
-// isOurOsArch returns true if s contains our OS and our Arch
-func isOurOsArch(s string) bool {
-	s = strings.ToLower(s)
-	check := func(base string, aliases map[string][]string) bool {
-		names := []string{base}
-		names = append(names, aliases[base]...)
-		for _, name := range names {
-			if strings.Contains(s, name) {
-				return true
-			}
-		}
-		return false
-	}
-	return check(runtime.GOARCH, archAliases) && check(runtime.GOOS, osAliases)
-}
-
 // get a file for download
 func getFile(url, fileName string) {
 	log.Printf("Downloading %q from %q", fileName, url)
@@ -327,66 +229,6 @@ func run(args ...string) {
 	}
 }
 
-// Untars fileName from srcFile
-func untar(srcFile, fileName, extractDir string) {
-	f, err := os.Open(srcFile)
-	if err != nil {
-		log.Fatalf("Couldn't open tar: %v", err)
-	}
-	defer func() {
-		err := f.Close()
-		if err != nil {
-			log.Fatalf("Couldn't close tar: %v", err)
-		}
-	}()
-
-	var in io.Reader = f
-
-	srcExt := filepath.Ext(srcFile)
-	if srcExt == ".gz" || srcExt == ".tgz" {
-		gzf, err := gzip.NewReader(f)
-		if err != nil {
-			log.Fatalf("Couldn't open gzip: %v", err)
-		}
-		in = gzf
-	}
-
-	tarReader := tar.NewReader(in)
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			log.Fatalf("Trouble reading tar file: %v", err)
-		}
-		name := header.Name
-		switch header.Typeflag {
-		case tar.TypeReg:
-			baseName := filepath.Base(name)
-			if baseName == fileName {
-				outPath := filepath.Join(extractDir, fileName)
-				out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
-				if err != nil {
-					log.Fatalf("Couldn't open output file: %v", err)
-				}
-				defer func() {
-					err := out.Close()
-					if err != nil {
-						log.Fatalf("Couldn't close output: %v", err)
-					}
-				}()
-				n, err := io.Copy(out, tarReader)
-				if err != nil {
-					log.Fatalf("Couldn't write output file: %v", err)
-				}
-				log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
-			}
-		}
-	}
-}
-
 func main() {
 	flag.Parse()
 	args := flag.Args()
@@ -402,12 +244,7 @@ func main() {
 		log.Fatalf("Invalid regexp for name %q: %v", nameRe, err)
 	}
 
-	var assetURL, assetName string
-	if *useAPI {
-		assetURL, assetName = getAsset(project, matchName)
-	} else {
-		assetURL, assetName = getAssetFromReleasesPage(project, matchName)
-	}
+	assetURL, assetName := getAsset(project, matchName)
 	fileName := filepath.Join(os.TempDir(), assetName)
 	getFile(assetURL, fileName)
 
@@ -420,6 +257,8 @@ func main() {
 			log.Fatalf("Need to set -bindir")
 		}
 		log.Printf("Unpacking %s from %s and installing into %s", *extract, fileName, *bindir)
-		untar(fileName, *extract, *bindir+"/")
+		run("tar", "xf", fileName, *extract)
+		run("chmod", "a+x", *extract)
+		run("mv", "-f", *extract, *bindir+"/")
 	}
 }

bin/make_changelog.py

@@ -12,8 +12,8 @@ from collections import defaultdict
 
 IGNORE_RES = [
     r"^Add .* to contributors$",
-    r"^Start v\d+\.\d+(\.\d+)?-DEV development$",
-    r"^Version v\d+\.\d+(\.\d+)?$",
+    r"^Start v\d+.\d+-DEV development$",
+    r"^Version v\d.\d+$",
 ]
 
 IGNORE_RE = re.compile("(?:" + "|".join(IGNORE_RES) + ")")

bin/make_manual.py

@@ -36,7 +36,6 @@ docs = [
     "http.md",
     "hubic.md",
     "jottacloud.md",
-    "koofr.md",
     "mega.md",
     "azureblob.md",
     "onedrive.md",

bin/tidy-beta

@@ -3,7 +3,7 @@
 
 version="$1"
 if [ "$version" = "" ]; then
-    echo "Syntax: $0 <version, eg v1.42> [delete]"
+    echo "Syntax: $0 <version> [delete]"
     exit 1
 fi
 dry_run="--dry-run"
@@ -14,4 +14,4 @@ else
     echo "Use '$0 $version delete' to actually delete files"
 fi
 
-rclone ${dry_run} -P --fast-list --checkers 16 --transfers 16 delete --include "**${version}**" memstore:beta-rclone-org
+rclone ${dry_run} --fast-list -P --checkers 16 --transfers 16 delete --include "**/${version}**" memstore:beta-rclone-org

bin/upload-github

@@ -29,7 +29,7 @@ github-release release \
     --name "rclone" \
     --description "Rclone - rsync for cloud storage. Sync files to and from many cloud storage providers."
 
-for build in `ls build | grep -v current | grep -v testbuilds`; do
+for build in `ls build | grep -v current`; do
     echo "Uploading ${build}"
     base="${build%.*}"
     parts=(${base//-/ })

cmd/about/about.go

@@ -95,9 +95,6 @@ Use the --json flag for a computer readable output, eg
 			if err != nil {
 				return errors.Wrap(err, "About call failed")
 			}
-			if u == nil {
-				return errors.New("nil usage returned")
-			}
 			if jsonOutput {
 				out := json.NewEncoder(os.Stdout)
 				out.SetIndent("", "\t")

cmd/cmd.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
-	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configflags"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/filter"
@@ -84,7 +83,7 @@ func NewFsFile(remote string) (fs.Fs, string) {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
-	f, err := cache.Get(remote)
+	f, err := fs.NewFs(remote)
 	switch err {
 	case fs.ErrorIsFile:
 		return f, path.Base(fsPath)
@@ -132,7 +131,7 @@ func NewFsSrc(args []string) fs.Fs {
 //
 // This must point to a directory
 func newFsDir(remote string) fs.Fs {
-	f, err := cache.Get(remote)
+	f, err := fs.NewFs(remote)
 	if err != nil {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
@@ -181,7 +180,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 			log.Fatalf("%q is a directory", args[1])
 		}
 	}
-	fdst, err := cache.Get(dstRemote)
+	fdst, err := fs.NewFs(dstRemote)
 	switch err {
 	case fs.ErrorIsFile:
 		fs.CountError(err)
@@ -231,34 +230,22 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	SigInfoHandler()
 	for try := 1; try <= *retries; try++ {
 		err = f()
-		fs.CountError(err)
-		lastErr := accounting.Stats.GetLastError()
-		if err == nil {
-			err = lastErr
-		}
-		if !Retry || !accounting.Stats.Errored() {
+		if !Retry || (err == nil && !accounting.Stats.Errored()) {
 			if try > 1 {
 				fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries)
 			}
 			break
 		}
-		if accounting.Stats.HadFatalError() {
+		if fserrors.IsFatalError(err) || accounting.Stats.HadFatalError() {
 			fs.Errorf(nil, "Fatal error received - not attempting retries")
 			break
 		}
-		if accounting.Stats.Errored() && !accounting.Stats.HadRetryError() {
+		if fserrors.IsNoRetryError(err) || (accounting.Stats.Errored() && !accounting.Stats.HadRetryError()) {
 			fs.Errorf(nil, "Can't retry this error - not attempting retries")
 			break
 		}
-		if retryAfter := accounting.Stats.RetryAfter(); !retryAfter.IsZero() {
-			d := retryAfter.Sub(time.Now())
-			if d > 0 {
-				fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
-				time.Sleep(d)
-			}
-		}
-		if lastErr != nil {
-			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), lastErr)
+		if err != nil {
+			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), err)
 		} else {
 			fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.Stats.GetErrors())
 		}
@@ -271,12 +258,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	}
 	stopStats()
 	if err != nil {
-		nerrs := accounting.Stats.GetErrors()
-		if nerrs <= 1 {
-			log.Printf("Failed to %s: %v", cmd.Name(), err)
-		} else {
-			log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, err)
-		}
+		log.Printf("Failed to %s: %v", cmd.Name(), err)
 		resolveExitCode(err)
 	}
 	if showStats && (accounting.Stats.Errored() || *statsInterval > 0) {
@@ -359,7 +341,8 @@ func initConfig() {
 	configflags.SetFlags()
 
 	// Load filters
-	err := filterflags.Reload()
+	var err error
+	filter.Active, err = filter.NewFilter(&filterflags.Opt)
 	if err != nil {
 		log.Fatalf("Failed to load filters: %v", err)
 	}
@@ -473,7 +456,7 @@ func AddBackendFlags() {
 					help = help[:nl]
 				}
 				help = strings.TrimSpace(help)
-				flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
+				flag := pflag.CommandLine.VarPF(opt, name, string(opt.ShortOpt), help)
 				if _, isBool := opt.Default.(bool); isBool {
 					flag.NoOptDefVal = "true"
 				}

cmd/cmount/mount.go

@@ -127,7 +127,7 @@ func waitFor(fn func() bool) (ok bool) {
 func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
-	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
+	// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
 	if runtime.GOOS != "windows" {
 		fi, err := os.Stat(mountpoint)
 		if err != nil {

cmd/config/config.go

@@ -98,9 +98,6 @@ Note that if the config process would normally ask a question the
 default is taken.  Each time that happens rclone will print a message
 saying how to affect the value taken.
 
-If any of the parameters passed is a password field, then rclone will
-automatically obscure them before putting them in the config file.
-
 So for example if you wanted to configure a Google Drive remote but
 using remote authorization you would do this:
 
@@ -128,14 +125,10 @@ var configUpdateCommand = &cobra.Command{
 Update an existing remote's options. The options should be passed in
 in pairs of <key> <value>.
 
-For example to update the env_auth field of a remote of name myremote
-you would do:
+For example to update the env_auth field of a remote of name myremote you would do:
 
     rclone config update myremote swift env_auth true
 
-If any of the parameters passed is a password field, then rclone will
-automatically obscure them before putting them in the config file.
-
 If the remote uses oauth the token will be updated, if you don't
 require this add an extra parameter thus:
 
@@ -175,9 +168,6 @@ should be passed in in pairs of <key> <value>.
 For example to set password of a remote of name myremote you would do:
 
     rclone config password myremote fieldname mypassword
-
-This command is obsolete now that "config update" and "config create"
-both support obscuring passwords directly.
 `,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(3, 256, command, args)

cmd/copy/copy.go

@@ -7,13 +7,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var (
-	createEmptySrcDirs = false
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
-	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy")
 }
 
 var commandDefintion = &cobra.Command{
@@ -74,7 +69,7 @@ changed recently very efficiently like this:
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc, createEmptySrcDirs)
+				return sync.CopyDir(fdst, fsrc)
 			}
 			return operations.CopyFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/copyto/copyto.go

@@ -48,7 +48,7 @@ destination.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc, false)
+				return sync.CopyDir(fdst, fsrc)
 			}
 			return operations.CopyFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/gendocs/gendocs.go

@@ -11,7 +11,6 @@ import (
 	"github.com/ncw/rclone/cmd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"github.com/spf13/pflag"
 )
 
 func init() {
@@ -51,10 +50,6 @@ rclone.org website.`,
 			base := strings.TrimSuffix(name, path.Ext(name))
 			return "/commands/" + strings.ToLower(base) + "/"
 		}
-		// Hide all of the root entries flags
-		cmd.Root.Flags().VisitAll(func(flag *pflag.Flag) {
-			flag.Hidden = true
-		})
 		return doc.GenMarkdownTreeCustom(cmd.Root, out, prepender, linkHandler)
 	},
 }

cmd/help.go

@@ -32,48 +32,8 @@ documentation, changelog and configuration walkthroughs.
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
 	},
-	BashCompletionFunction: bashCompletionFunc,
 }
 
-const (
-	bashCompletionFunc = `
-__rclone_custom_func() {
-    if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
-        local cur cword prev words
-        if declare -F _init_completion > /dev/null; then
-            _init_completion -n : || return
-        else
-            __rclone_init_completion -n : || return
-        fi
-        if [[ $cur != *:* ]]; then
-            local remote
-            while IFS= read -r remote; do
-                [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
-            done < <(command rclone listremotes)
-            if [[ ${COMPREPLY[@]} ]]; then
-                local paths=("$cur"*)
-                [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
-            fi
-        else
-            local path=${cur#*:}
-            if [[ $path == */* ]]; then
-                local prefix=$(eval printf '%s' "${path%/*}")
-            else
-                local prefix=
-            fi
-            local line
-            while IFS= read -r line; do
-                local reply=${prefix:+$prefix/}$line
-                [[ $reply != $path* ]] || COMPREPLY+=("$reply")
-            done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
-	    [[ ! ${COMPREPLY[@]} ]] || compopt -o filenames
-        fi
-        [[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
-    fi
-}
-`
-)
-
 // root help command
 var helpCommand = &cobra.Command{
 	Use:   "help",

cmd/info/info.go

@@ -21,22 +21,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-type position int
-
-const (
-	positionMiddle position = 1 << iota
-	positionLeft
-	positionRight
-	positionNone position = 0
-	positionAll  position = positionRight<<1 - 1
-)
-
 var (
 	checkNormalization bool
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
-	positionList       = []position{positionMiddle, positionLeft, positionRight}
 )
 
 func init() {
@@ -70,7 +59,7 @@ a bit of go code for each one.
 type results struct {
 	f                    fs.Fs
 	mu                   sync.Mutex
-	stringNeedsEscaping  map[string]position
+	charNeedsEscaping    map[rune]bool
 	maxFileLength        int
 	canWriteUnnormalized bool
 	canReadUnnormalized  bool
@@ -80,8 +69,8 @@ type results struct {
 
 func newResults(f fs.Fs) *results {
 	return &results{
-		f:                   f,
-		stringNeedsEscaping: make(map[string]position),
+		f:                 f,
+		charNeedsEscaping: make(map[rune]bool),
 	}
 }
 
@@ -90,13 +79,13 @@ func (r *results) Print() {
 	fmt.Printf("// %s\n", r.f.Name())
 	if checkControl {
 		escape := []string{}
-		for c, needsEscape := range r.stringNeedsEscaping {
-			if needsEscape != positionNone {
+		for c, needsEscape := range r.charNeedsEscaping {
+			if needsEscape {
 				escape = append(escape, fmt.Sprintf("0x%02X", c))
 			}
 		}
 		sort.Strings(escape)
-		fmt.Printf("stringNeedsEscaping = []byte{\n")
+		fmt.Printf("charNeedsEscaping = []byte{\n")
 		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
 		fmt.Printf("}\n")
 	}
@@ -141,45 +130,20 @@ func (r *results) checkUTF8Normalization() {
 	}
 }
 
-func (r *results) checkStringPositions(s string) {
-	fs.Infof(r.f, "Writing position file 0x%0X", s)
-	positionError := positionNone
-
-	for _, pos := range positionList {
-		path := ""
-		switch pos {
-		case positionMiddle:
-			path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
-		case positionLeft:
-			path = fmt.Sprintf("%s-position-left-%0X", s, s)
-		case positionRight:
-			path = fmt.Sprintf("position-right-%0X-%s", s, s)
-		default:
-			panic("invalid position: " + pos.String())
-		}
-		_, writeErr := r.writeFile(path)
-		if writeErr != nil {
-			fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
-		} else {
-			fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
-		}
-		obj, getErr := r.f.NewObject(path)
-		if getErr != nil {
-			fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
-		} else {
-			if obj.Size() != 50 {
-				fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
-			} else {
-				fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
-			}
-		}
-		if writeErr != nil || getErr != nil {
-			positionError += pos
-		}
+// check we can write file with the rune passed in
+func (r *results) checkChar(c rune) {
+	fs.Infof(r.f, "Writing file 0x%02X", c)
+	path := fmt.Sprintf("0x%02X-%c-", c, c)
+	_, err := r.writeFile(path)
+	escape := false
+	if err != nil {
+		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
+		escape = true
+	} else {
+		fs.Infof(r.f, "OK writing file 0x%02X", c)
 	}
-
 	r.mu.Lock()
-	r.stringNeedsEscaping[s] = positionError
+	r.charNeedsEscaping[c] = escape
 	r.mu.Unlock()
 }
 
@@ -193,28 +157,19 @@ func (r *results) checkControls() {
 	}
 	var wg sync.WaitGroup
 	for i := rune(0); i < 128; i++ {
-		s := string(i)
 		if i == 0 || i == '/' {
 			// We're not even going to check NULL or /
-			r.stringNeedsEscaping[s] = positionAll
+			r.charNeedsEscaping[i] = true
 			continue
 		}
 		wg.Add(1)
-		go func(s string) {
+		c := i
+		go func() {
 			defer wg.Done()
 			token := <-tokens
-			r.checkStringPositions(s)
+			r.checkChar(c)
 			tokens <- token
-		}(s)
-	}
-	for _, s := range []string{"＼", "\xBF", "\xFE"} {
-		wg.Add(1)
-		go func(s string) {
-			defer wg.Done()
-			token := <-tokens
-			r.checkStringPositions(s)
-			tokens <- token
-		}(s)
+		}()
 	}
 	wg.Wait()
 	fs.Infof(r.f, "Done trying to create control character file names")
@@ -313,35 +268,3 @@ func readInfo(f fs.Fs) error {
 	r.Print()
 	return nil
 }
-
-func (e position) String() string {
-	switch e {
-	case positionNone:
-		return "none"
-	case positionAll:
-		return "all"
-	}
-	var buf bytes.Buffer
-	if e&positionMiddle != 0 {
-		buf.WriteString("middle")
-		e &= ^positionMiddle
-	}
-	if e&positionLeft != 0 {
-		if buf.Len() != 0 {
-			buf.WriteRune(',')
-		}
-		buf.WriteString("left")
-		e &= ^positionLeft
-	}
-	if e&positionRight != 0 {
-		if buf.Len() != 0 {
-			buf.WriteRune(',')
-		}
-		buf.WriteString("right")
-		e &= ^positionRight
-	}
-	if e != positionNone {
-		panic("invalid position")
-	}
-	return buf.String()
-}

cmd/info/process.sh

@@ -1,40 +0,0 @@
-set -euo pipefail
-
-for f in info-*.log; do 
-  for pos in middle left right; do
-    egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos
-    egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos
-  done
-  {
-    echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}"
-    echo "Write\tWrite\tWrite\tGet\tGet\tGet"
-    echo "Mid\tLeft\tRight\tMid\tLeft\tRight"
-    paste $f.write_{middle,left,right} $f.get_{middle,left,right}
-  } > $f.csv
-done
-
-for f in info-*.list; do
-  for pos in middle left right; do
-    cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos
-  done
-  {
-    echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}"
-    echo "List\tList\tList"
-    echo "Mid\tLeft\tRight"
-    for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do
-      echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss)
-      echo -n "\t"
-      echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/'    $f.left   | tr -d "\t\r" | grep -a . || echo Miss)
-      echo -n "\t"
-      echo    $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/'   $f.right  | tr -d "\t\r" | grep -a . || echo Miss)
-      # echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t"
-      # echo -n $(grep -a "position-left-$e"    $f.left   | tr -d "\t\r" || echo Miss)"\t"
-      # echo    $(grep -a "position-right-$e-"  $f.right  | tr -d "\t\r" || echo Miss)
-    done
-    } > $f.csv
-done
-
-for f in info-*.list; do 
-    paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv
-done
-paste *.full.csv > info-complete.csv