Version v1.43.1

- Reduced size of the social menu and increased the size of the content
- Added scrollable property to the index menus
- Fixed code wrapping issue

Fixes #2103

Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>

.appveyor.yml

@@ -4,9 +4,6 @@ os: Windows Server 2012 R2
 
 clone_folder: c:\gopath\src\github.com\ncw\rclone
 
-cache:
-  - '%LocalAppData%\go-build'
-
 environment:
   GOPATH: C:\gopath
   CPATH: C:\Program Files (x86)\WinFsp\inc\fuse
@@ -46,4 +43,4 @@ artifacts:
 - path: build/*-v*.zip
 
 deploy_script:
-- IF "%APPVEYOR_REPO_NAME%" == "ncw/rclone" IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload
+- IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload

.circleci/config.yml

@@ -1,4 +1,3 @@
----
 version: 2
 
 jobs:
@@ -14,10 +13,10 @@ jobs:
     - run:
         name: Cross-compile rclone
         command: |
-            docker pull rclone/xgo-cgofuse
+            docker pull billziss/xgo-cgofuse
             go get -v github.com/karalabe/xgo
             xgo \
-                --image=rclone/xgo-cgofuse \
+                --image=billziss/xgo-cgofuse \
                 --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
                 -tags cmount \
                 .
@@ -30,21 +29,6 @@ jobs:
         command: |
             mkdir -p /tmp/rclone.dist
             cp -R rclone-* /tmp/rclone.dist
-            mkdir build
-            cp -R rclone-* build/
-
-    - run:
-        name: Build rclone
-        command: |
-            go version
-            go build
-
-    - run:
-        name: Upload artifacts
-        command: |
-          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
-              make circleci_upload
-          fi
 
     - store_artifacts:
         path: /tmp/rclone.dist

.github/ISSUE_TEMPLATE.md

@@ -1,31 +0,0 @@
-<!--
-
-Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
-
-If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
-
-    https://forum.rclone.org/
-
-instead of filing an issue for a quick response.
-
-If you are reporting a bug or asking for a new feature then please use one of the templates here:
-
-    https://github.com/ncw/rclone/issues/new
-
-otherwise fill in the form below.
-
-Thank you
-
-The Rclone Developers
-
--->
-
-
-#### Output of `rclone version`
-
-
-
-#### Describe the issue
-
-
-

.github/ISSUE_TEMPLATE/Feature.md

@@ -1,36 +0,0 @@
----
-name: Feature request
-about: Suggest a new feature or enhancement for rclone
----
-
-<!--
-
-Welcome :-)
-
-So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
-
-Here is a checklist of things to do:
-
-  1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum first: https://forum.rclone.org/
-  3. Make a feature request issue (this is the right place!).
-  4. Be prepared to get involved making the feature :-)
-
-Looking forward to your great idea!
-
-The Rclone Developers
-
--->
-
-
-#### What is your current rclone version (output from `rclone version`)?
-
-
-
-#### What problem are you are trying to solve?
-
-
-
-#### How do you think rclone should be changed to solve that?
-
-

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,29 +0,0 @@
-<!--
-Thank you very much for contributing code or documentation to rclone! Please
-fill out the following questions to make it easier for us to review your
-changes.
-
-You do not need to check all the boxes below all at once, feel free to take
-your time and add more commits. If you're done and ready for review, please
-check the last box.
--->
-
-#### What is the purpose of this change?
-
-<!--
-Describe the changes here
--->
-
-#### Was the change discussed in an issue or in the forum before?
-
-<!--
-Link issues and relevant forum posts here.
--->
-
-#### Checklist
-
-- [ ] I have read the [contribution guidelines](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
-- [ ] I have added tests for all changes in this PR if appropriate.
-- [ ] I have added documentation for the changes if appropriate.
-- [ ] All commit messages are in [house style](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#commit-messages).
-- [ ] I'm done, this Pull Request is ready for review :-)

.golangci.yml

@@ -1,26 +0,0 @@
-# golangci-lint configuration options
-
-linters:
-  enable:
-    - deadcode
-    - errcheck
-    - goimports
-    - golint
-    - ineffassign
-    - structcheck
-    - varcheck
-    - govet
-    - unconvert
-    #- prealloc
-    #- maligned
-  disable-all: true
-
-issues:
-  # Enable some lints excluded by default
-  exclude-use-default: false
-
-  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
-
-  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
-  max-same-issues: 0

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.travis.yml

@@ -1,106 +1,51 @@
----
 language: go
 sudo: required
-dist: xenial
+dist: trusty
 os:
-  - linux
-go_import_path: github.com/ncw/rclone
+- linux
+go:
+- 1.7.x
+- 1.8.x
+- 1.9.x
+- 1.10.x
+- 1.11.x
+- tip
 before_install:
-  - git fetch --unshallow --tags
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-      choco install -y winfsp zip make
-      cd ../..                  # fix crlf in git checkout
-      mv $TRAVIS_REPO_SLUG _old
-      git config --global core.autocrlf false
-      git clone _old $TRAVIS_REPO_SLUG
-      cd $TRAVIS_REPO_SLUG
-    fi
+- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
+- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
 install:
-  - make vars
+- git fetch --unshallow --tags
+- make vars
+- make build_dep
+script:
+- make check
+- make quicktest
+- make compile_all
 env:
   global:
     - GOTAGS=cmount
-    - GO111MODULE=off
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
     - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-      - fuse
-      - libfuse-dev
-      - rpm
-      - pkg-config
-cache:
-  directories:
-    - $HOME/.cache/go-build
+    - fuse
+    - libfuse-dev
+    - rpm
+    - pkg-config
 matrix:
   allow_failures:
-    - go: tip
+  - go: tip
   include:
-    - go: 1.9.x
-      script:
-        - make quicktest
-    - go: 1.10.x
-      script:
-        - make quicktest
-    - go: 1.11.x
-      script:
-        - make quicktest
-    - go: 1.12.x
-      env:
-        - GOTAGS=cmount
-      script:
-        - make build_dep
-        - make check
-        - make quicktest
-        - make racequicktest
-        - make compile_all
-    - os: osx
-      go: 1.12.x
-      env:
-        - GOTAGS=  # cmount doesn't work on osx travis for some reason
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-      script:
-        - make
-        - make quicktest
-        - make racequicktest
-    # - os: windows
-    #   go: 1.12.x
-    #   env:
-    #     - GOTAGS=cmount
-    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
-    #   #filter_secrets: false     # works around a problem with secrets under windows
-    #   cache:
-    #     directories:
-    #       - ${LocalAppData}/go-build
-    #   script:
-    #     - make
-    #     - make quicktest
-    #     - make racequicktest
-    - go: tip
-      script:
-        - make quicktest
-
+  - os: osx
+    go: 1.11.x
+    env: GOTAGS=""
 deploy:
   provider: script
   script: make travis_beta
   skip_cleanup: true
   on:
-    repo: ncw/rclone
     all_branches: true
-    go: 1.12.x
-    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"
+    go: 1.11.x
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -21,14 +21,14 @@ with the [latest beta of rclone](https://beta.rclone.org/):
 ## Submitting a pull request ##
 
 If you find a bug that you'd like to fix, or a new feature that you'd
-like to implement then please submit a pull request via GitHub.
+like to implement then please submit a pull request via Github.
 
 If it is a big feature then make an issue first so it can be discussed.
 
 You'll need a Go environment set up with GOPATH set.  See [the Go
 getting started docs](https://golang.org/doc/install) for more info.
 
-First in your web browser press the fork button on [rclone's GitHub
+First in your web browser press the fork button on [rclone's Github
 page](https://github.com/ncw/rclone).
 
 Now in your terminal
@@ -64,31 +64,22 @@ packages which you can install with
 
 Make sure you
 
-  * Add [documentation](#writing-documentation) for a new feature.
-  * Follow the [commit message guidelines](#commit-messages).
-  * Add [unit tests](#testing) for a new feature
+  * Add documentation for a new feature (see below for where)
+  * Add unit tests for a new feature
   * squash commits down to one per feature
-  * rebase to master with `git rebase master`
+  * rebase to master `git rebase master`
 
 When you are done with that
 
-    git push origin my-new-feature
+  git push origin my-new-feature
 
-Go to the GitHub website and click [Create pull
+Go to the Github website and click [Create pull
 request](https://help.github.com/articles/creating-a-pull-request/).
 
 You patch will get reviewed and you might get asked to fix some stuff.
 
 If so, then make the changes in the same branch, squash the commits,
-rebase it to master then push it to GitHub with `--force`.
-
-## Enabling CI for your fork ##
-
-The CI config files for rclone have taken care of forks of the project, so you can enable CI for your fork repo easily.
-
-rclone currently uses [Travis CI](https://travis-ci.org/), [AppVeyor](https://ci.appveyor.com/), and
-[Circle CI](https://circleci.com/) to build the project. To enable them for your fork, simply go into their
-websites, find your fork of rclone, and enable building there.
+rebase it to master then push it to Github with `--force`.
 
 ## Testing ##
 
@@ -123,19 +114,12 @@ but they can be run against any of the remotes.
     cd fs/operations
     go test -v -remote TestDrive:
 
-If you want to use the integration test framework to run these tests
-all together with an HTML report and test retries then from the
-project root:
-
-    go install github.com/ncw/rclone/fstest/test_all
-    test_all -backend drive
-
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
 
     make test
 
-This command is run daily on the integration test server. You can
+This command is run daily on the the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##
@@ -189,14 +173,10 @@ with modules beneath.
 
 If you are adding a new feature then please update the documentation.
 
-If you add a new general flag (not for a backend), then document it in
+If you add a new flag, then if it is a general flag, document it in
 `docs/content/docs.md` - the flags there are supposed to be in
-alphabetical order.
-
-If you add a new backend option/flag, then it should be documented in
-the source file in the `Help:` field.  The first line of this is used
-for the flag help, the remainder is shown to the user in `rclone
-config` and is added to the docs with `make backenddocs`.
+alphabetical order.  If it is a remote specific flag, then document it
+in `docs/content/remote.md`.
 
 The only documentation you need to edit are the `docs/content/*.md`
 files.  The MANUAL.*, rclone.1, web site etc are all auto generated
@@ -215,20 +195,14 @@ file.
 ## Commit messages ##
 
 Please make the first line of your commit message a summary of the
-change that a user (not a developer) of rclone would like to read, and
-prefix it with the directory of the change followed by a colon.  The
-changelog gets made by looking at just these first lines so make it
-good!
+change, and prefix it with the directory of the change followed by a
+colon.  The changelog gets made by looking at just these first lines
+so make it good!
 
 If you have more to say about the commit, then enter a blank line and
 carry on the description.  Remember to say why the change was needed -
 the commit itself shows what was changed.
 
-Writing more is better than less.  Comparing the behaviour before the
-change to that after the change is very useful.  Imagine you are
-writing to yourself in 12 months time when you've forgotten everything
-about what you just did and you need to get up to speed quickly.
-
 If the change fixes an issue then write `Fixes #1234` in the commit
 message.  This can be on the subject line if it will fit.  If you
 don't want to close the associated issue just put `#1234` and the
@@ -276,8 +250,9 @@ To add a dependency `github.com/ncw/new_dependency` see the
 instructions below.  These will fetch the dependency, add it to
 `go.mod` and `go.sum` and vendor it for older go versions.
 
-    GO111MODULE=on go get github.com/ncw/new_dependency
-    GO111MODULE=on go mod vendor
+    export GO111MODULE=on
+    go get github.com/ncw/new_dependency
+    go mod vendor
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
@@ -292,8 +267,9 @@ in `vendor`.
 
 If you need to update a dependency then run
 
-    GO111MODULE=on go get -u github.com/pkg/errors
-    GO111MODULE=on go mod vendor
+    export GO111MODULE=on
+    go get -u github.com/pkg/errors
+    go mod vendor
 
 Check in in a single commit as above.
 
@@ -350,13 +326,7 @@ Unit tests
 
 Integration tests
 
-  * Add your backend to `fstest/test_all/config.yaml`
-      * Once you've done that then you can use the integration test framework from the project root:
-      * go install ./...
-      * test_all -backend remote
-
-Or if you want to run the integration tests manually:
-
+  * Add your fs to `fstest/test_all/test_all.go`
   * Make sure integration tests pass with
       * `cd fs/operations`
       * `go test -v -remote TestRemote:`
@@ -371,10 +341,11 @@ See the [testing](#testing) section for more information on integration tests.
 
 Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in alphabetical order but with the local file system last.
 
-  * `README.md` - main GitHub page
-  * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+  * `README.md` - main Github page
+  * `docs/content/remote.md` - main docs page
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
   * `docs/content/about.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
+  * `cmd/cmd.go` - the main help for rclone

ISSUE_TEMPLATE.md

@@ -1,17 +1,14 @@
----
-name: Bug report
-about: Report a problem with rclone
----
-
 <!--
 
-Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
+Hi!
 
-If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
+We understand you are having a problem with rclone or have an idea for an improvement - we want to help you with that!
+
+If you've just got a question or aren't sure if you've found a bug then please use the rclone forum
 
     https://forum.rclone.org/
 
-instead of filing an issue for a quick response.
+instead of filing an issue.  We'll reply quickly and it won't increase our massive issue backlog.
 
 If you think you might have found a bug, please can you try to replicate it with the latest beta?
 
@@ -19,7 +16,9 @@ If you think you might have found a bug, please can you try to replicate it with
     
 If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
 
-Thank you
+If you have an idea for an improvement, then please search the old issues first and if you don't find your idea, make a new issue.
+
+Thanks
 
 The Rclone Developers
 
@@ -28,23 +27,17 @@ The Rclone Developers
 #### What is the problem you are having with rclone?
 
 
-
-#### What is your rclone version (output from `rclone version`)
-
+#### What is your rclone version (eg output from `rclone -V`)
 
 
 #### Which OS you are using and how many bits (eg Windows 7, 64 bit)
 
 
-
 ####  Which cloud storage system are you using? (eg Google Drive)
 
 
-
 #### The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
 
 
-
 #### A log from the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
 
-

MAINTAINERS.md

@@ -1,17 +1,12 @@
 # Maintainers guide for rclone #
 
-Current active maintainers of rclone are:
+Current active maintainers of rclone are
 
-| Name             | GitHub ID   | Specific Responsibilities    |
-| :--------------- | :---------- | :--------------------------  |
-| Nick Craig-Wood  | @ncw        | overall project health       |
-| Stefan Breunig   | @breunigs   |                              |
-| Ishuah Kariuki   | @ishuah     |                              |
-| Remus Bunduc     | @remusb     | cache backend                |
-| Fabian Möller    | @B4dM4n     |                              |
-| Alex Chen        | @Cnly       | onedrive backend             |
-| Sandeep Ummadi   | @sandeepkru | azureblob backend            |
-| Sebastian Bünger | @buengese   | jottacloud & yandex backends |
+  * Nick Craig-Wood @ncw
+  * Stefan Breunig @breunigs
+  * Ishuah Kariuki @ishuah
+  * Remus Bunduc @remusb - cache subsystem maintainer
+  * Fabian Möller @B4dM4n
 
 **This is a work in progress Draft**
 
@@ -61,7 +56,7 @@ Close tickets as soon as you can - make sure they are tagged with a release.  Po
 
 Try to process pull requests promptly!
 
-Merging pull requests on GitHub itself works quite well now-a-days so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
+Merging pull requests on Github itself works quite well now-a-days so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
 
 After merging the commit, in your local master branch, do `git pull` then run `bin/update-authors.py` to update the authors file then `git push`.
 

Makefile

@@ -11,12 +11,14 @@ ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	BRANCH_PATH :=
 endif
 TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 ifneq ($(TAG),$(LAST_TAG))
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+# Run full tests if go >= go1.11
+FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -24,7 +26,6 @@ BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
-LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
 .PHONY: rclone vars version
@@ -41,6 +42,7 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
+	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -48,26 +50,46 @@ version:
 
 # Full suite of integration tests
 test:	rclone
-	go install --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/ncw/rclone/fstest/test_all
-	-test_all 2>&1 | tee test_all.log
-	@echo "Written logs in test_all.log"
+	go install github.com/ncw/rclone/fstest/test_all
+	-go test -v -count 1 $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log
+	-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log
+	@echo "Written logs in test.log and fs/test_all.log"
 
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-
-racequicktest:
+ifdef FULL_TESTS
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
+endif
 
 # Do source code quality checks
 check:	rclone
-	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@golangci-lint run $(LINTTAGS) ./...
-	@echo "-- END CODE QUALITY REPORT ---------------------------------"
+ifdef FULL_TESTS
+	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	errcheck $(BUILDTAGS) ./...
+	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
+	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+else
+	@echo Skipping source quality tests as version of go too old
+endif
+
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+ifdef FULL_TESTS
+	go get -u github.com/kisielk/errcheck
+	go get -u golang.org/x/tools/cmd/goimports
+	go get -u github.com/golang/lint/golint
+endif
 
 # Get the release dependencies
 release_dep:
@@ -77,15 +99,15 @@ release_dep:
 # Update dependencies
 update:
 	GO111MODULE=on go get -u ./...
-	GO111MODULE=on go mod tidy
-	GO111MODULE=on go mod vendor
+	GO111MODULE=on go tidy
+	GO111MODULE=on go vendor
 
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
 	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
@@ -95,10 +117,7 @@ MANUAL.txt:	MANUAL.md
 	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
-
-backenddocs: rclone bin/make_backend_docs.py
-	./bin/make_backend_docs.py
+	rclone gendocs docs/content/commands/
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -133,8 +152,8 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
+	rclone -v copy --exclude '*current*' build/ memstore:downloads-rclone-org/$(TAG)
+	rclone -v copy --include '*current*' --include version.txt build/ memstore:downloads-rclone-org
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -151,7 +170,11 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
+ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+else
+	@echo Skipping compile all as version of go too old
+endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -160,26 +183,14 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-circleci_upload:
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
-ifndef BRANCH_PATH
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
-endif
-	@echo Beta release ready at $(BETA_URL)/testbuilds
-
 BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
-ifeq ($(TRAVIS_OS_NAME),windows)
-# BUILD_FLAGS := -include "^windows/" -cgo
-# 386 doesn't build yet
-	BUILD_FLAGS := -include "^windows/amd64" -cgo
-endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
@@ -191,7 +202,7 @@ endif
 
 # Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -v sync $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server -v -w
@@ -218,3 +229,4 @@ startdev:
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
+

README.md

@@ -2,96 +2,61 @@
 
 [Website](https://rclone.org) |
 [Documentation](https://rclone.org/docs/) |
-[Download](https://rclone.org/downloads/) | 
 [Contributing](CONTRIBUTING.md) |
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
-[Forum](https://forum.rclone.org/) | 
+[Forum](https://forum.rclone.org/)
+[G+](https://google.com/+RcloneOrg)
 
 [![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
 [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
 [![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
-[![Go Report Card](https://goreportcard.com/badge/github.com/ncw/rclone)](https://goreportcard.com/report/github.com/ncw/rclone)
 [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
-# Rclone
+Rclone is a command line program to sync files and directories to and from
 
-Rclone *("rsync for cloud storage")* is a command line program to sync files and directories to and from different cloud storage providers.
+  * Amazon Drive ([See note](https://rclone.org/amazonclouddrive/#status))
+  * Amazon S3 / Dreamhost / Ceph / Minio / Wasabi
+  * Backblaze B2
+  * Box
+  * Dropbox
+  * FTP
+  * Google Cloud Storage
+  * Google Drive
+  * HTTP
+  * Hubic
+  * Jottacloud
+  * Mega
+  * Microsoft Azure Blob Storage
+  * Microsoft OneDrive
+  * OpenDrive
+  * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
+  * pCloud
+  * QingStor
+  * SFTP
+  * Webdav / Owncloud / Nextcloud
+  * Yandex Disk
+  * The local filesystem
 
-## Storage providers
+Features
 
-  * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
-  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
-  * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
-  * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
-  * Box [:page_facing_up:](https://rclone.org/box/)
-  * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
-  * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
-  * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
-  * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
-  * FTP [:page_facing_up:](https://rclone.org/ftp/)
-  * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
-  * Google Drive [:page_facing_up:](https://rclone.org/drive/)
-  * HTTP [:page_facing_up:](https://rclone.org/http/)
-  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
-  * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
-  * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
-  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
-  * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
-  * Mega [:page_facing_up:](https://rclone.org/mega/)
-  * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
-  * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
-  * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
-  * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
-  * OVH [:page_facing_up:](https://rclone.org/swift/)
-  * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
-  * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
-  * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
-  * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
-  * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
-  * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
-  * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
-  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
-  * SFTP [:page_facing_up:](https://rclone.org/sftp/)
-  * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
-  * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
-  * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
-  * The local filesystem [:page_facing_up:](https://rclone.org/local/)
-  
-Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
-
-## Features
-
-  * MD5/SHA-1 hashes checked at all times for file integrity
+  * MD5/SHA1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
-  * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
-  * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
-  * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, e.g. two different cloud accounts
-  * Optional encryption ([Crypt](https://rclone.org/crypt/))
-  * Optional cache ([Cache](https://rclone.org/cache/))
-  * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
+  * Copy mode to just copy new/changed files
+  * Sync (one way) mode to make a directory identical
+  * Check mode to check for file hash equality
+  * Can sync to and from network, eg two different cloud accounts
+  * Optional encryption (Crypt)
+  * Optional FUSE mount
 
-## Installation & documentation
+See the home page for installation, usage, documentation, changelog
+and configuration walkthroughs.
 
-Please see the [rclone website](https://rclone.org/) for:
-
-  * [Installation](https://rclone.org/install/)
-  * [Documentation & configuration](https://rclone.org/docs/)
-  * [Changelog](https://rclone.org/changelog/)
-  * [FAQ](https://rclone.org/faq/)
-  * [Storage providers](https://rclone.org/overview/)
-  * [Forum](https://forum.rclone.org/)
-  * ...and more
-
-## Downloads
-
-  * https://rclone.org/downloads/
+  * https://rclone.org/
 
 License
 -------
 
 This is free software under the terms of MIT the license (check the
-[COPYING file](/COPYING) included in this package).
+COPYING file included in this package).

RELEASE.md

@@ -11,7 +11,7 @@ Making a release
   * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX.0"
+  * git commit -a -v -m "Version v1.XX"
   * make retag
   * git push --tags origin master
   * # Wait for the appveyor and travis builds to complete then...
@@ -27,45 +27,7 @@ Making a release
 
 Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
-      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files
   * git commit -a -v
-
-If `make update` fails with errors like this:
-
-```
-# github.com/cpuguy83/go-md2man/md2man
-../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:11:16: undefined: blackfriday.EXTENSION_NO_INTRA_EMPHASIS
-../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:12:16: undefined: blackfriday.EXTENSION_TABLES
-```
-
-Can be fixed with
-
-    * GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
-    * GO111MODULE=on go mod tidy
-    * GO111MODULE=on go mod vendor
- 
-
-Making a point release.  If rclone needs a point release due to some
-horrendous bug, then
-  * git branch v1.XX v1.XX-fixes
-  * git cherry-pick any fixes
-  * Test (see above)
-  * make NEW_TAG=v1.XX.1 tag
-  * edit docs/content/changelog.md
-  * make TAG=v1.43.1 doc
-  * git commit -a -v -m "Version v1.XX.1"
-  * git tag -d -v1.XX.1
-  * git tag -s -m "Version v1.XX.1" v1.XX.1
-  * git push --tags -u origin v1.XX-fixes
-  * make BRANCH_PATH= TAG=v1.43.1 fetch_binaries
-  * make TAG=v1.43.1 tarball
-  * make TAG=v1.43.1 sign_upload
-  * make TAG=v1.43.1 check_sign
-  * make TAG=v1.43.1 upload
-  * make TAG=v1.43.1 upload_website
-  * make TAG=v1.43.1 upload_github
-  * NB this overwrites the current beta so after the release, rebuild the last travis build
-  * Announce!

backend/alias/alias.go

@@ -2,12 +2,13 @@ package alias
 
 import (
 	"errors"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/fspath"
 )
 
 // Register with Fs
@@ -30,7 +31,7 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs constructs an Fs from the path.
+// NewFs contstructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -46,9 +47,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote)
+	_, configName, fsPath, err := fs.ParseRemote(opt.Remote)
 	if err != nil {
 		return nil, err
 	}
-	return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
+	root = path.Join(fsPath, filepath.ToSlash(root))
+	if configName == "local" {
+		return fs.NewFs(root)
+	}
+	return fs.NewFs(configName + ":" + root)
 }

backend/alias/alias_internal_test.go

@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}

backend/all/all.go

@@ -16,7 +16,6 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/jottacloud"
-	_ "github.com/ncw/rclone/backend/koofr"
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"
@@ -26,7 +25,6 @@ import (
 	_ "github.com/ncw/rclone/backend/s3"
 	_ "github.com/ncw/rclone/backend/sftp"
 	_ "github.com/ncw/rclone/backend/swift"
-	_ "github.com/ncw/rclone/backend/union"
 	_ "github.com/ncw/rclone/backend/webdav"
 	_ "github.com/ncw/rclone/backend/yandex"
 )

backend/amazonclouddrive/amazonclouddrive.go

@@ -21,7 +21,7 @@ import (
 	"strings"
 	"time"
 
-	acd "github.com/ncw/go-acd"
+	"github.com/ncw/go-acd"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
@@ -32,6 +32,7 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
+	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -96,42 +97,13 @@ func init() {
 			Hide:     fs.OptionHideBoth,
 			Advanced: true,
 		}, {
-			Name: "upload_wait_per_gb",
-			Help: `Additional time per GB to wait after a failed complete upload to see if it appears.
-
-Sometimes Amazon Drive gives an error when a file has been fully
-uploaded but the file appears anyway after a little while.  This
-happens sometimes for files over 1GB in size and nearly every time for
-files bigger than 10GB. This parameter controls the time rclone waits
-for the file to appear.
-
-The default value for this parameter is 3 minutes per GB, so by
-default it will wait 3 minutes for every GB uploaded to see if the
-file appears.
-
-You can disable this feature by setting it to 0. This may cause
-conflict errors as rclone retries the failed upload but the file will
-most likely appear correctly eventually.
-
-These values were determined empirically by observing lots of uploads
-of big files for a range of file sizes.
-
-Upload with the "-v" flag to see more info about what rclone is doing
-in this situation.`,
+			Name:     "upload_wait_per_gb",
+			Help:     "Additional time per GB to wait after a failed complete upload to see if it appears.",
 			Default:  fs.Duration(180 * time.Second),
 			Advanced: true,
 		}, {
-			Name: "templink_threshold",
-			Help: `Files >= this size will be downloaded via their tempLink.
-
-Files this size or more will be downloaded via their "tempLink". This
-is to work around a problem with Amazon Drive which blocks downloads
-of files bigger than about 10GB.  The default for this is 9GB which
-shouldn't need to be changed.
-
-To download files above this threshold, rclone requests a "tempLink"
-which downloads the file through a temporary URL directly from the
-underlying S3 storage.`,
+			Name:     "templink_threshold",
+			Help:     "Files >= this size will be downloaded via their tempLink.",
 			Default:  defaultTempLinkThreshold,
 			Advanced: true,
 		}},
@@ -154,7 +126,7 @@ type Fs struct {
 	noAuthClient *http.Client       // unauthenticated http client
 	root         string             // the path we are working on
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	trueRootID   string             // ID of true root directory
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
@@ -263,7 +235,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, acdConfig, baseClient)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure Amazon Drive")
+		log.Fatalf("Failed to configure Amazon Drive: %v", err)
 	}
 
 	c := acd.NewClient(oAuthClient)
@@ -272,7 +244,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:         root,
 		opt:          *opt,
 		c:            c,
-		pacer:        fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
+		pacer:        pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
 		noAuthClient: fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -311,16 +283,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		// Assume it is a file
 		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, f.trueRootID, &tempF)
-		tempF.root = newRoot
+		newF := *f
+		newF.dirCache = dircache.New(newRoot, f.trueRootID, &newF)
+		newF.root = newRoot
 		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(false)
+		err = newF.dirCache.FindRoot(false)
 		if err != nil {
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(remote, nil)
+		_, err := newF.newObjectWithInfo(remote, nil)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -328,13 +300,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			}
 			return nil, err
 		}
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/ncw/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
 		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
+		return &newF, fs.ErrorIsFile
 	}
 	return f, nil
 }
@@ -1092,7 +1059,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		if !bigObject {
 			in, resp, err = file.OpenHeaders(headers)
 		} else {
-			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
+			in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers)
 		}
 		return o.fs.shouldRetry(resp, err)
 	})
@@ -1273,38 +1240,24 @@ func (o *Object) MimeType() string {
 // Automatically restarts itself in case of unexpected behaviour of the remote.
 //
 // Close the returned channel to stop being notified.
-func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
 	checkpoint := f.opt.Checkpoint
 
+	quit := make(chan bool)
 	go func() {
-		var ticker *time.Ticker
-		var tickerC <-chan time.Time
 		for {
+			checkpoint = f.changeNotifyRunner(notifyFunc, checkpoint)
+			if err := config.SetValueAndSave(f.name, "checkpoint", checkpoint); err != nil {
+				fs.Debugf(f, "Unable to save checkpoint: %v", err)
+			}
 			select {
-			case pollInterval, ok := <-pollIntervalChan:
-				if !ok {
-					if ticker != nil {
-						ticker.Stop()
-					}
-					return
-				}
-				if pollInterval == 0 {
-					if ticker != nil {
-						ticker.Stop()
-						ticker, tickerC = nil, nil
-					}
-				} else {
-					ticker = time.NewTicker(pollInterval)
-					tickerC = ticker.C
-				}
-			case <-tickerC:
-				checkpoint = f.changeNotifyRunner(notifyFunc, checkpoint)
-				if err := config.SetValueAndSave(f.name, "checkpoint", checkpoint); err != nil {
-					fs.Debugf(f, "Unable to save checkpoint: %v", err)
-				}
+			case <-quit:
+				return
+			case <-time.After(pollInterval):
 			}
 		}
 	}()
+	return quit
 }
 
 func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoint string) string {

backend/azureblob/azureblob.go

@@ -1,13 +1,12 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris
+// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
 
 package azureblob
 
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/hex"
@@ -22,14 +21,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Azure/azure-pipeline-go/pipeline"
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-storage-blob-go/2018-03-28/azblob"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/fserrors"
-	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/pacer"
@@ -40,19 +37,18 @@ const (
 	minSleep              = 10 * time.Millisecond
 	maxSleep              = 10 * time.Second
 	decayConstant         = 1    // bigger for slower decay, exponential
-	maxListChunkSize      = 5000 // number of items to read at once
+	listChunkSize         = 5000 // number of items to read at once
 	modTimeKey            = "mtime"
 	timeFormatIn          = time.RFC3339
 	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
 	maxTotalParts         = 50000 // in multipart upload
 	storageDefaultBaseURL = "blob.core.windows.net"
 	// maxUncommittedSize = 9 << 30 // can't upload bigger than this
-	defaultChunkSize    = 4 * fs.MebiByte
-	maxChunkSize        = 100 * fs.MebiByte
-	defaultUploadCutoff = 256 * fs.MebiByte
-	maxUploadCutoff     = 256 * fs.MebiByte
+	defaultChunkSize    = 4 * 1024 * 1024
+	maxChunkSize        = 100 * 1024 * 1024
+	defaultUploadCutoff = 256 * 1024 * 1024
+	maxUploadCutoff     = 256 * 1024 * 1024
 	defaultAccessTier   = azblob.AccessTierNone
-	maxTryTimeout       = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
 )
 
 // Register with Fs
@@ -76,44 +72,18 @@ func init() {
 			Advanced: true,
 		}, {
 			Name:     "upload_cutoff",
-			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
-			Default:  defaultUploadCutoff,
+			Help:     "Cutoff for switching to chunked upload.",
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
-			Name: "chunk_size",
-			Help: `Upload chunk size (<= 100MB).
-
-Note that this is stored in memory and there may be up to
-"--transfers" chunks stored at once in memory.`,
-			Default:  defaultChunkSize,
-			Advanced: true,
-		}, {
-			Name: "list_chunk",
-			Help: `Size of blob list.
-
-This sets the number of blobs requested in each listing chunk. Default
-is the maximum, 5000. "List blobs" requests are permitted 2 minutes
-per megabyte to complete. If an operation is taking longer than 2
-minutes per megabyte on average, it will time out (
-[source](https://docs.microsoft.com/en-us/rest/api/storageservices/setting-timeouts-for-blob-service-operations#exceptions-to-default-timeout-interval)
-). This can be used to limit the number of blobs items to return, to
-avoid the time out.`,
-			Default:  maxListChunkSize,
+			Name:     "chunk_size",
+			Help:     "Upload chunk size. Must fit in memory.",
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}, {
 			Name: "access_tier",
-			Help: `Access tier of blob: hot, cool or archive.
-
-Archived blobs can be restored by setting access tier to hot or
-cool. Leave blank if you intend to use default access tier, which is
-set at account level
-
-If there is no "access tier" specified, rclone doesn't apply any tier.
-rclone performs "Set Tier" operation on blobs while uploading, if objects
-are not modified, specifying "access tier" to new one will have no effect.
-If blobs are in "archive tier" at remote, trying to perform data transfer
-operations from remote will not be allowed. User should first restore by
-tiering blob to "Hot" or "Cool".`,
+			Help: "Access tier of blob, supports hot, cool and archive tiers.\nArchived blobs can be restored by setting access tier to hot or cool." +
+				" Leave blank if you intend to use default access tier, which is set at account level",
 			Advanced: true,
 		}},
 	})
@@ -121,14 +91,13 @@ tiering blob to "Hot" or "Cool".`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account       string        `config:"account"`
-	Key           string        `config:"key"`
-	Endpoint      string        `config:"endpoint"`
-	SASURL        string        `config:"sas_url"`
-	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
-	ListChunkSize uint          `config:"list_chunk"`
-	AccessTier    string        `config:"access_tier"`
+	Account      string        `config:"account"`
+	Key          string        `config:"key"`
+	Endpoint     string        `config:"endpoint"`
+	SASURL       string        `config:"sas_url"`
+	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
+	AccessTier   string        `config:"access_tier"`
 }
 
 // Fs represents a remote azure server
@@ -137,14 +106,13 @@ type Fs struct {
 	root             string                // the path we are working on if any
 	opt              Options               // parsed config options
 	features         *fs.Features          // optional features
-	client           *http.Client          // http client we are using
 	svcURL           *azblob.ServiceURL    // reference to serviceURL
 	cntURL           *azblob.ContainerURL  // reference to containerURL
 	container        string                // the container we are working on
 	containerOKMu    sync.Mutex            // mutex to protect container OK
 	containerOK      bool                  // true if we have created the container
 	containerDeleted bool                  // true if we have deleted the container
-	pacer            *fs.Pacer             // To pace and retry the API calls
+	pacer            *pacer.Pacer          // To pace and retry the API calls
 	uploadToken      *pacer.TokenDispenser // control concurrency
 }
 
@@ -203,19 +171,6 @@ func parsePath(path string) (container, directory string, err error) {
 	return
 }
 
-// validateAccessTier checks if azureblob supports user supplied tier
-func validateAccessTier(tier string) bool {
-	switch tier {
-	case string(azblob.AccessTierHot),
-		string(azblob.AccessTierCool),
-		string(azblob.AccessTierArchive):
-		// valid cases
-		return true
-	default:
-		return false
-	}
-}
-
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	401, // Unauthorized (eg "Token has expired")
@@ -241,73 +196,7 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	const minChunkSize = fs.Byte
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	if cs > maxChunkSize {
-		return errors.Errorf("%s is greater than %s", cs, maxChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxUploadCutoff {
-		return errors.Errorf("%v must be less than or equal to %v", cs, maxUploadCutoff)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
-// httpClientFactory creates a Factory object that sends HTTP requests
-// to a rclone's http.Client.
-//
-// copied from azblob.newDefaultHTTPClientFactory
-func httpClientFactory(client *http.Client) pipeline.Factory {
-	return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
-		return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
-			r, err := client.Do(request.WithContext(ctx))
-			if err != nil {
-				err = pipeline.NewError(err, "HTTP request failed")
-			}
-			return pipeline.NewHTTPResponse(r), err
-		}
-	})
-}
-
-// newPipeline creates a Pipeline using the specified credentials and options.
-//
-// this code was copied from azblob.NewPipeline
-func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
-	// Closest to API goes first; closest to the wire goes last
-	factories := []pipeline.Factory{
-		azblob.NewTelemetryPolicyFactory(o.Telemetry),
-		azblob.NewUniqueRequestIDPolicyFactory(),
-		azblob.NewRetryPolicyFactory(o.Retry),
-		c,
-		pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked
-		azblob.NewRequestLogPolicyFactory(o.RequestLog),
-	}
-	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
-}
-
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -316,16 +205,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, errors.Wrap(err, "azure: upload cutoff")
+	if opt.UploadCutoff > maxUploadCutoff {
+		return nil, errors.Errorf("azure: upload cutoff (%v) must be less than or equal to %v", opt.UploadCutoff, maxUploadCutoff)
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "azure: chunk size")
-	}
-	if opt.ListChunkSize > maxListChunkSize {
-		return nil, errors.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
+	if opt.ChunkSize > maxChunkSize {
+		return nil, errors.Errorf("azure: chunk size can't be greater than %v - was %v", maxChunkSize, opt.ChunkSize)
 	}
 	container, directory, err := parsePath(root)
 	if err != nil {
@@ -337,28 +221,17 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	if opt.AccessTier == "" {
 		opt.AccessTier = string(defaultAccessTier)
-	} else if !validateAccessTier(opt.AccessTier) {
-		return nil, errors.Errorf("Azure Blob: Supported access tiers are %s, %s and %s",
-			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
+	} else {
+		switch opt.AccessTier {
+		case string(azblob.AccessTierHot):
+		case string(azblob.AccessTierCool):
+		case string(azblob.AccessTierArchive):
+			// valid cases
+		default:
+			return nil, errors.Errorf("azure: Supported access tiers are %s, %s and %s", string(azblob.AccessTierHot), string(azblob.AccessTierCool), azblob.AccessTierArchive)
+		}
 	}
 
-	f := &Fs{
-		name:        name,
-		opt:         *opt,
-		container:   container,
-		root:        directory,
-		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
-		client:      fshttp.NewClient(fs.Config),
-	}
-	f.features = (&fs.Features{
-		ReadMimeType:  true,
-		WriteMimeType: true,
-		BucketBased:   true,
-		SetTier:       true,
-		GetTier:       true,
-	}).Fill(f)
-
 	var (
 		u            *url.URL
 		serviceURL   azblob.ServiceURL
@@ -366,16 +239,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	)
 	switch {
 	case opt.Account != "" && opt.Key != "":
-		credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
-		if err != nil {
-			return nil, errors.Wrapf(err, "Failed to parse credentials")
-		}
-
+		credential := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
 		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint")
 		}
-		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+		pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
 		serviceURL = azblob.NewServiceURL(*u, pipeline)
 		containerURL = serviceURL.NewContainerURL(container)
 	case opt.SASURL != "":
@@ -384,7 +253,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to parse SAS URL")
 		}
 		// use anonymous credentials in case of sas url
-		pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+		pipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})
 		// Check if we have container level SAS or account level sas
 		parts := azblob.NewBlobURLParts(*u)
 		if parts.ContainerName != "" {
@@ -392,7 +261,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
 			}
 
-			f.container = parts.ContainerName
+			container = parts.ContainerName
 			containerURL = azblob.NewContainerURL(*u, pipeline)
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -401,9 +270,22 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	default:
 		return nil, errors.New("Need account+key or connectionString or sasURL")
 	}
-	f.svcURL = &serviceURL
-	f.cntURL = &containerURL
 
+	f := &Fs{
+		name:        name,
+		opt:         *opt,
+		container:   container,
+		root:        directory,
+		svcURL:      &serviceURL,
+		cntURL:      &containerURL,
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+	}
+	f.features = (&fs.Features{
+		ReadMimeType:  true,
+		WriteMimeType: true,
+		BucketBased:   true,
+	}).Fill(f)
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the (container,directory) is actually an existing file
@@ -417,8 +299,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		_, err := f.NewObject(remote)
 		if err != nil {
-			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
-				// File doesn't exist or is a directory so return old f
+			if err == fs.ErrorObjectNotFound {
+				// File doesn't exist so return old f
 				f.root = oldRoot
 				return f, nil
 			}
@@ -474,21 +356,6 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
 	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
 }
 
-// Returns whether file is a directory marker or not
-func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool {
-	// Directory markers are 0 length
-	if size == 0 {
-		// Note that metadata with hdi_isfolder = true seems to be a
-		// defacto standard for marking blobs as directories.
-		endsWithSlash := strings.HasSuffix(remote, "/")
-		if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" {
-			return true
-		}
-
-	}
-	return false
-}
-
 // listFn is called from list to handle an object
 type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error
 
@@ -524,7 +391,6 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 		MaxResults: int32(maxResults),
 	}
 	ctx := context.Background()
-	directoryMarkers := map[string]struct{}{}
 	for marker := (azblob.Marker{}); marker.NotDone(); {
 		var response *azblob.ListBlobsHierarchySegmentResponse
 		err := f.pacer.Call(func() (bool, error) {
@@ -554,23 +420,13 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 				continue
 			}
 			remote := file.Name[len(f.root):]
-			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
-				if strings.HasSuffix(remote, "/") {
-					remote = remote[:len(remote)-1]
-				}
-				err = fn(remote, file, true)
-				if err != nil {
-					return err
-				}
-				// Keep track of directory markers. If recursing then
-				// there will be no Prefixes so no need to keep track
-				if !recurse {
-					directoryMarkers[remote] = struct{}{}
-				}
-				continue // skip directory marker
+			// Check for directory
+			isDirectory := strings.HasSuffix(remote, "/")
+			if isDirectory {
+				remote = remote[:len(remote)-1]
 			}
 			// Send object
-			err = fn(remote, file, false)
+			err = fn(remote, file, isDirectory)
 			if err != nil {
 				return err
 			}
@@ -583,10 +439,6 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 				continue
 			}
 			remote = remote[len(f.root):]
-			// Don't send if already sent as a directory marker
-			if _, found := directoryMarkers[remote]; found {
-				continue
-			}
 			// Send object
 			err = fn(remote, nil, true)
 			if err != nil {
@@ -622,7 +474,7 @@ func (f *Fs) markContainerOK() {
 
 // listDir lists a single directory
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
-	err = f.list(dir, false, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
+	err = f.list(dir, false, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
 		entry, err := f.itemToDirEntry(remote, object, isDirectory)
 		if err != nil {
 			return err
@@ -693,7 +545,7 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 		return fs.ErrorListBucketRequired
 	}
 	list := walk.NewListRHelper(callback)
-	err = f.list(dir, true, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
+	err = f.list(dir, true, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
 		entry, err := f.itemToDirEntry(remote, object, isDirectory)
 		if err != nil {
 			return err
@@ -714,11 +566,11 @@ type listContainerFn func(*azblob.ContainerItem) error
 // listContainersToFn lists the containers to the function supplied
 func (f *Fs) listContainersToFn(fn listContainerFn) error {
 	params := azblob.ListContainersSegmentOptions{
-		MaxResults: int32(f.opt.ListChunkSize),
+		MaxResults: int32(listChunkSize),
 	}
 	ctx := context.Background()
 	for marker := (azblob.Marker{}); marker.NotDone(); {
-		var response *azblob.ListContainersSegmentResponse
+		var response *azblob.ListContainersResponse
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
 			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
@@ -754,35 +606,6 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
-// Check if the container exists
-//
-// NB this can return incorrect results if called immediately after container deletion
-func (f *Fs) dirExists() (bool, error) {
-	options := azblob.ListBlobsSegmentOptions{
-		Details: azblob.BlobListingDetails{
-			Copy:             false,
-			Metadata:         false,
-			Snapshots:        false,
-			UncommittedBlobs: false,
-			Deleted:          false,
-		},
-		MaxResults: 1,
-	}
-	err := f.pacer.Call(func() (bool, error) {
-		ctx := context.Background()
-		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
-		return f.shouldRetry(err)
-	})
-	if err == nil {
-		return true, nil
-	}
-	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
-	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
-		return false, nil
-	}
-	return false, err
-}
-
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	f.containerOKMu.Lock()
@@ -790,15 +613,6 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
-	if !f.containerDeleted {
-		exists, err := f.dirExists()
-		if err == nil {
-			f.containerOK = exists
-		}
-		if err != nil || exists {
-			return err
-		}
-	}
 
 	// now try to create the container
 	err := f.pacer.Call(func() (bool, error) {
@@ -811,11 +625,6 @@ func (f *Fs) Mkdir(dir string) error {
 					f.containerOK = true
 					return false, nil
 				case azblob.ServiceCodeContainerBeingDeleted:
-					// From https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container
-					// When a container is deleted, a container with the same name cannot be created
-					// for at least 30 seconds; the container may not be available for more than 30
-					// seconds if the service is still processing the request.
-					time.Sleep(6 * time.Second) // default 10 retries will be 60 seconds
 					f.containerDeleted = true
 					return true, err
 				}
@@ -833,7 +642,7 @@ func (f *Fs) Mkdir(dir string) error {
 // isEmpty checks to see if a given directory is empty and returns an error if not
 func (f *Fs) isEmpty(dir string) (err error) {
 	empty := true
-	err = f.list(dir, true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
+	err = f.list("", true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
 		empty = false
 		return nil
 	})
@@ -943,7 +752,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	var startCopy *azblob.BlobStartCopyFromURLResponse
 
 	err = f.pacer.Call(func() (bool, error) {
-		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options)
+		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, options, options)
 		return f.shouldRetry(err)
 	})
 	if err != nil {
@@ -1028,37 +837,26 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 //  o.md5
 //  o.meta
 func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
-	metadata := info.NewMetadata()
-	size := info.ContentLength()
-	if isDirectoryMarker(size, metadata, o.remote) {
-		return fs.ErrorNotAFile
-	}
-	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
-	// this as base64 encoded string.
+	// NOTE - In BlobGetPropertiesResponse, Client library returns MD5 as base64 decoded string
+	// unlike BlobProperties in BlobItem (used in decodeMetadataFromBlob) which returns base64
+	// encoded bytes. Object needs to maintain this as base64 encoded string.
 	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
 	o.mimeType = info.ContentType()
-	o.size = size
-	o.modTime = info.LastModified()
+	o.size = info.ContentLength()
+	o.modTime = time.Time(info.LastModified())
 	o.accessTier = azblob.AccessTierType(info.AccessTier())
-	o.setMetadata(metadata)
+	o.setMetadata(info.NewMetadata())
 
 	return nil
 }
 
 func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) {
-	metadata := info.Metadata
-	size := *info.Properties.ContentLength
-	if isDirectoryMarker(size, metadata, o.remote) {
-		return fs.ErrorNotAFile
-	}
-	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
-	// this as base64 encoded string.
-	o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5)
+	o.md5 = string(info.Properties.ContentMD5)
 	o.mimeType = *info.Properties.ContentType
-	o.size = size
+	o.size = *info.Properties.ContentLength
 	o.modTime = info.Properties.LastModified
 	o.accessTier = info.Properties.AccessTier
-	o.setMetadata(metadata)
+	o.setMetadata(info.Metadata)
 	return nil
 }
 
@@ -1104,6 +902,12 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
+// timeString returns modTime as the number of milliseconds
+// elapsed since January 1, 1970 UTC as a decimal string.
+func timeString(modTime time.Time) string {
+	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
+}
+
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1321,14 +1125,11 @@ outer:
 			defer o.fs.uploadToken.Put()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
 
-			// Upload the block, with MD5 for check
-			md5sum := md5.Sum(buf)
-			transactionalMD5 := md5sum[:]
 			err = o.fs.pacer.Call(func() (bool, error) {
 				bufferReader := bytes.NewReader(buf)
 				wrappedReader := wrap(bufferReader)
 				rs := readSeeker{wrappedReader, bufferReader}
-				_, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac, transactionalMD5)
+				_, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac)
 				return o.fs.shouldRetry(err)
 			})
 
@@ -1386,16 +1187,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	blob := o.getBlobReference()
 	httpHeaders := azblob.BlobHTTPHeaders{}
 	httpHeaders.ContentType = fs.MimeType(o)
-	// Compute the Content-MD5 of the file, for multiparts uploads it
-	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
-	// 		 in order to validate its integrity during transport
-	if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
-		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-		if err == nil {
-			httpHeaders.ContentMD5 = sourceMD5bytes
-		} else {
-			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+	// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
+	// MD5 only for PutBlob requests
+	if size < int64(o.fs.opt.UploadCutoff) {
+		if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				httpHeaders.ContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
 		}
 	}
 
@@ -1405,20 +1206,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		Metadata:        o.meta,
 		BlobHTTPHeaders: httpHeaders,
 	}
-	// FIXME Until https://github.com/Azure/azure-storage-blob-go/pull/75
-	// is merged the SDK can't upload a single blob of exactly the chunk
-	// size, so upload with a multpart upload to work around.
-	// See: https://github.com/ncw/rclone/issues/2653
-	multipartUpload := size >= int64(o.fs.opt.UploadCutoff)
-	if size == int64(o.fs.opt.ChunkSize) {
-		multipartUpload = true
-		fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size)
-	}
 
 	ctx := context.Background()
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		if multipartUpload {
+		if size >= int64(o.fs.opt.UploadCutoff) {
 			// If a large file upload in chunks
 			err = o.uploadMultipart(in, size, &blob, &httpHeaders)
 		} else {
@@ -1444,7 +1236,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	}
 
 	// Now, set blob tier based on configured access tier
-	return o.SetTier(o.fs.opt.AccessTier)
+	desiredAccessTier := azblob.AccessTierType(o.fs.opt.AccessTier)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, err := blob.SetTier(ctx, desiredAccessTier)
+		return o.fs.shouldRetry(err)
+	})
+
+	if err != nil {
+		return errors.Wrap(err, "Failed to set Blob Tier")
+	}
+	return nil
 }
 
 // Remove an object
@@ -1469,41 +1270,6 @@ func (o *Object) AccessTier() azblob.AccessTierType {
 	return o.accessTier
 }
 
-// SetTier performs changing object tier
-func (o *Object) SetTier(tier string) error {
-	if !validateAccessTier(tier) {
-		return errors.Errorf("Tier %s not supported by Azure Blob Storage", tier)
-	}
-
-	// Check if current tier already matches with desired tier
-	if o.GetTier() == tier {
-		return nil
-	}
-	desiredAccessTier := azblob.AccessTierType(tier)
-	blob := o.getBlobReference()
-	ctx := context.Background()
-	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
-		return o.fs.shouldRetry(err)
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "Failed to set Blob Tier")
-	}
-
-	// Set access tier on local object also, this typically
-	// gets updated on get blob properties
-	o.accessTier = desiredAccessTier
-	fs.Debugf(o, "Successfully changed object tier to %s", tier)
-
-	return nil
-}
-
-// GetTier returns object tier in azure as string
-func (o *Object) GetTier() string {
-	return string(o.accessTier)
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs        = &Fs{}

backend/azureblob/azureblob_internal_test.go

@@ -1,18 +0,0 @@
-// +build !plan9,!solaris
-
-package azureblob
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func (f *Fs) InternalTest(t *testing.T) {
-	// Check first feature flags are set on this
-	// remote
-	enabled := f.Features().SetTier
-	assert.True(t, enabled)
-	enabled = f.Features().GetTier
-	assert.True(t, enabled)
-}

backend/azureblob/azureblob_test.go

@@ -1,37 +1,20 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris
+// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
 
-package azureblob
+package azureblob_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/azureblob"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
-		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MaxChunkSize: maxChunkSize,
-		},
+		RemoteName: "TestAzureBlob:",
+		NilObject:  (*azureblob.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris
+// +build freebsd netbsd openbsd plan9 solaris !go1.8
 
 package azureblob

backend/b2/api/types.go

@@ -17,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal satisfies the Fatal interface
+// Fatal statisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
 	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is uninitialized
+// IsZero returns true if the timestamp is unitialised
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }
@@ -136,7 +136,6 @@ type AuthorizeAccountResponse struct {
 	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
 		BucketID     string      `json:"bucketId"`     // When present, access is restricted to one bucket.
-		BucketName   string      `json:"bucketName"`   // When present, name of bucket - may be empty
 		Capabilities []string    `json:"capabilities"` // A list of strings, each one naming a capability the key has.
 		NamePrefix   interface{} `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`

backend/b2/b2.go

@@ -48,9 +48,9 @@ const (
 	decayConstant       = 1 // bigger for slower decay, exponential
 	maxParts            = 10000
 	maxVersions         = 100 // maximum number of versions we search in --b2-versions mode
-	minChunkSize        = 5 * fs.MebiByte
-	defaultChunkSize    = 96 * fs.MebiByte
-	defaultUploadCutoff = 200 * fs.MebiByte
+	minChunkSize        = 5E6
+	defaultChunkSize    = 96 * 1024 * 1024
+	defaultUploadCutoff = 200E6
 )
 
 // Globals
@@ -77,24 +77,14 @@ func init() {
 			Help:     "Endpoint for the service.\nLeave blank normally.",
 			Advanced: true,
 		}, {
-			Name: "test_mode",
-			Help: `A flag string for X-Bz-Test-Mode header for debugging.
-
-This is for debugging purposes only. Setting it to one of the strings
-below will cause b2 to return specific errors:
-
-  * "fail_some_uploads"
-  * "expire_some_account_authorization_tokens"
-  * "force_cap_exceeded"
-
-These will be set in the "X-Bz-Test-Mode" header which is documented
-in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).`,
+			Name:     "test_mode",
+			Help:     "A flag string for X-Bz-Test-Mode header for debugging.",
 			Default:  "",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
 		}, {
 			Name:     "versions",
-			Help:     "Include old versions in directory listings.\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
+			Help:     "Include old versions in directory listings.",
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -102,36 +92,14 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 			Help:    "Permanently delete files on remote removal, otherwise hide files.",
 			Default: false,
 		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload.
-
-Files above this size will be uploaded in chunks of "--b2-chunk-size".
-
-This value should be set no larger than 4.657GiB (== 5GB).`,
-			Default:  defaultUploadCutoff,
+			Name:     "upload_cutoff",
+			Help:     "Cutoff for switching to chunked upload.",
+			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
-			Name: "chunk_size",
-			Help: `Upload chunk size. Must fit in memory.
-
-When uploading large files, chunk the file into this size.  Note that
-these chunks are buffered in memory and there might a maximum of
-"--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimum size.`,
-			Default:  defaultChunkSize,
-			Advanced: true,
-		}, {
-			Name:     "disable_checksum",
-			Help:     `Disable checksums for large (> upload cutoff) files`,
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name: "download_url",
-			Help: `Custom endpoint for downloads.
-
-This is usually set to a Cloudflare CDN URL as Backblaze offers
-free egress for data downloaded through the Cloudflare network.
-Leave blank if you want to use the endpoint provided by Backblaze.`,
+			Name:     "chunk_size",
+			Help:     "Upload chunk size. Must fit in memory.",
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}},
 	})
@@ -139,16 +107,14 @@ Leave blank if you want to use the endpoint provided by Backblaze.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account         string        `config:"account"`
-	Key             string        `config:"key"`
-	Endpoint        string        `config:"endpoint"`
-	TestMode        string        `config:"test_mode"`
-	Versions        bool          `config:"versions"`
-	HardDelete      bool          `config:"hard_delete"`
-	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
-	DisableCheckSum bool          `config:"disable_checksum"`
-	DownloadURL     string        `config:"download_url"`
+	Account      string        `config:"account"`
+	Key          string        `config:"key"`
+	Endpoint     string        `config:"endpoint"`
+	TestMode     string        `config:"test_mode"`
+	Versions     bool          `config:"versions"`
+	HardDelete   bool          `config:"hard_delete"`
+	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
 }
 
 // Fs represents a remote b2 server
@@ -167,7 +133,7 @@ type Fs struct {
 	uploadMu      sync.Mutex                   // lock for upload variable
 	uploads       []*api.GetUploadURLResponse  // result of get upload URL calls
 	authMu        sync.Mutex                   // lock for authorizing the account
-	pacer         *fs.Pacer                    // To pace and retry the API calls
+	pacer         *pacer.Pacer                 // To pace and retry the API calls
 	bufferTokens  chan []byte                  // control concurrency of multipart uploads
 }
 
@@ -251,7 +217,13 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 				fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err)
 			}
 		}
-		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Second)
+		retryAfterDuration := time.Duration(retryAfter) * time.Second
+		if f.pacer.GetSleep() < retryAfterDuration {
+			fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err)
+			// We set 1/2 the value here because the pacer will double it immediately
+			f.pacer.SetSleep(retryAfterDuration / 2)
+		}
+		return true, err
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -291,38 +263,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-		f.fillBufferTokens() // reset the buffer tokens
-	}
-	return
-}
-
-func checkUploadCutoff(opt *Options, cs fs.SizeSuffix) error {
-	if cs < opt.ChunkSize {
-		return errors.Errorf("%v is less than chunk size %v", cs, opt.ChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(&f.opt, cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -330,13 +271,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = checkUploadCutoff(opt, opt.UploadCutoff)
-	if err != nil {
-		return nil, errors.Wrap(err, "b2: upload cutoff")
+	if opt.UploadCutoff < opt.ChunkSize {
+		return nil, errors.Errorf("b2: upload cutoff (%v) must be greater than or equal to chunk size (%v)", opt.UploadCutoff, opt.ChunkSize)
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "b2: chunk size")
+	if opt.ChunkSize < minChunkSize {
+		return nil, errors.Errorf("b2: chunk size can't be less than %v - was %v", minChunkSize, opt.ChunkSize)
 	}
 	bucket, directory, err := parsePath(root)
 	if err != nil {
@@ -352,12 +291,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt.Endpoint = defaultEndpoint
 	}
 	f := &Fs{
-		name:   name,
-		opt:    *opt,
-		bucket: bucket,
-		root:   directory,
-		srv:    rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		name:         name,
+		opt:          *opt,
+		bucket:       bucket,
+		root:         directory,
+		srv:          rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
+		pacer:        pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		bufferTokens: make(chan []byte, fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -370,20 +310,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		f.srv.SetHeader(testModeHeader, testMode)
 		fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
 	}
-	f.fillBufferTokens()
+	// Fill up the buffer tokens
+	for i := 0; i < fs.Config.Transfers; i++ {
+		f.bufferTokens <- nil
+	}
 	err = f.authorizeAccount()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to authorize account")
 	}
 	// If this is a key limited to a single bucket, it must exist already
 	if f.bucket != "" && f.info.Allowed.BucketID != "" {
-		allowedBucket := f.info.Allowed.BucketName
-		if allowedBucket == "" {
-			return nil, errors.New("bucket that application key is restricted to no longer exists")
-		}
-		if allowedBucket != f.bucket {
-			return nil, errors.Errorf("you must use bucket %q with this application key", allowedBucket)
-		}
 		f.markBucketOK()
 		f.setBucketID(f.info.Allowed.BucketID)
 	}
@@ -485,14 +421,6 @@ func (f *Fs) clearUploadURL() {
 	f.uploadMu.Unlock()
 }
 
-// Fill up (or reset) the buffer tokens
-func (f *Fs) fillBufferTokens() {
-	f.bufferTokens = make(chan []byte, fs.Config.Transfers)
-	for i := 0; i < fs.Config.Transfers; i++ {
-		f.bufferTokens <- nil
-	}
-}
-
 // getUploadBlock gets a block from the pool of size chunkSize
 func (f *Fs) getUploadBlock() []byte {
 	buf := <-f.bufferTokens
@@ -952,13 +880,6 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
-		if apiErr, ok := err.(*api.Error); ok {
-			if apiErr.Code == "already_hidden" {
-				// sometimes eventual consistency causes this, so
-				// ignore this error since it is harmless
-				return nil
-			}
-		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1003,12 +924,6 @@ func (f *Fs) purge(oldOnly bool) error {
 			errReturn = err
 		}
 	}
-	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		if time.Since(time.Time(timestamp)).Hours() > 24 {
-			return true
-		}
-		return false
-	}
 
 	// Delete Config.Transfers in parallel
 	toBeDeleted := make(chan *api.File, fs.Config.Transfers)
@@ -1032,9 +947,6 @@ func (f *Fs) purge(oldOnly bool) error {
 				if object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
 					toBeDeleted <- object
-				} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
-					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
-					toBeDeleted <- object
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
 				}
@@ -1213,7 +1125,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return nil
+		return err
 	}
 	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
 	return nil
@@ -1306,17 +1218,9 @@ var _ io.ReadCloser = &openFile{}
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	opts := rest.Opts{
 		Method:  "GET",
+		RootURL: o.fs.info.DownloadURL,
 		Options: options,
 	}
-
-	// Use downloadUrl from backblaze if downloadUrl is not set
-	// otherwise use the custom downloadUrl
-	if o.fs.opt.DownloadURL == "" {
-		opts.RootURL = o.fs.info.DownloadURL
-	} else {
-		opts.RootURL = o.fs.opt.DownloadURL
-	}
-
 	// Download by id if set otherwise by name
 	if o.id != "" {
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
@@ -1477,7 +1381,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// Content-Type b2/x-auto to automatically set the stored Content-Type
 	// post upload. In the case where a file extension is absent or the
 	// lookup fails, the Content-Type is set to application/octet-stream. The
-	// Content-Type mappings can be pursued here.
+	// Content-Type mappings can be purused here.
 	//
 	// X-Bz-Content-Sha1
 	// required
@@ -1524,6 +1428,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		},
 		ContentLength: &size,
 	}
+	// for go1.8 (see release notes) we must nil the Body if we want a
+	// "Content-Length: 0" header which b2 requires for all files.
+	if size == 0 {
+		opts.Body = nil
+	}
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {

backend/b2/b2_test.go

@@ -1,10 +1,10 @@
 // Test B2 filesystem interface
-package b2
+package b2_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/b2"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -12,23 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestB2:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize:       minChunkSize,
-			NeedMultipleChunks: true,
-		},
+		NilObject:  (*b2.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/b2/upload.go

@@ -116,10 +116,8 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
 		},
 	}
 	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
-		}
+	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+		request.Info[sha1Key] = calculatedSha1
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {

backend/box/api/types.go

@@ -45,7 +45,7 @@ type Error struct {
 	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,11 +57,11 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo
-var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link"
+var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status"
 
 // Types of things in Item
 const (
@@ -86,10 +86,6 @@ type Item struct {
 	ContentCreatedAt  Time    `json:"content_created_at"`
 	ContentModifiedAt Time    `json:"content_modified_at"`
 	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
-	SharedLink        struct {
-		URL    string `json:"url,omitempty"`
-		Access string `json:"access,omitempty"`
-	} `json:"shared_link"`
 }
 
 // ModTime returns the modification time of the item
@@ -149,14 +145,6 @@ type CopyFile struct {
 	Parent Parent `json:"parent"`
 }
 
-// CreateSharedLink is the request for Public Link
-type CreateSharedLink struct {
-	SharedLink struct {
-		URL    string `json:"url,omitempty"`
-		Access string `json:"access,omitempty"`
-	} `json:"shared_link"`
-}
-
 // UploadSessionRequest is uses in Create Upload Session
 type UploadSessionRequest struct {
 	FolderID string `json:"folder_id,omitempty"` // don't pass for update

backend/box/box.go

@@ -85,7 +85,7 @@ func init() {
 			Help: "Box App Client Secret\nLeave blank normally.",
 		}, {
 			Name:     "upload_cutoff",
-			Help:     "Cutoff for switching to multipart upload (>= 50MB).",
+			Help:     "Cutoff for switching to multipart upload.",
 			Default:  fs.SizeSuffix(defaultUploadCutoff),
 			Advanced: true,
 		}, {
@@ -111,7 +111,7 @@ type Fs struct {
 	features     *fs.Features          // optional features
 	srv          *rest.Client          // the connection to the one drive server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *fs.Pacer             // pacer for API calls
+	pacer        *pacer.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
 }
@@ -126,7 +126,6 @@ type Object struct {
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
 	id          string    // ID of the object
-	publicLink  string    // Public Link for the object
 	sha1        string    // SHA-1 of the object content
 }
 
@@ -171,13 +170,13 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
+	authRety := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
+		authRety = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // substitute reserved characters for box
@@ -252,7 +251,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	root = parsePath(root)
 	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure Box")
+		log.Fatalf("Failed to configure Box: %v", err)
 	}
 
 	f := &Fs{
@@ -260,7 +259,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:        root,
 		opt:         *opt,
 		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{
@@ -283,16 +282,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		// Assume it is a file
 		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
-		tempF.root = newRoot
+		newF := *f
+		newF.dirCache = dircache.New(newRoot, rootID, &newF)
+		newF.root = newRoot
 		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(false)
+		err = newF.dirCache.FindRoot(false)
 		if err != nil {
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(remote, nil)
+		_, err := newF.newObjectWithInfo(remote, nil)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -300,14 +299,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			}
 			return nil, err
 		}
-		f.features.Fill(&tempF)
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/ncw/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
 		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
+		return &newF, fs.ErrorIsFile
 	}
 	return f, nil
 }
@@ -530,10 +523,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -851,46 +844,6 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	return nil
 }
 
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(remote string) (string, error) {
-	id, err := f.dirCache.FindDir(remote, false)
-	var opts rest.Opts
-	if err == nil {
-		fs.Debugf(f, "attempting to share directory '%s'", remote)
-
-		opts = rest.Opts{
-			Method:     "PUT",
-			Path:       "/folders/" + id,
-			Parameters: fieldsValue(),
-		}
-	} else {
-		fs.Debugf(f, "attempting to share single file '%s'", remote)
-		o, err := f.NewObject(remote)
-		if err != nil {
-			return "", err
-		}
-
-		if o.(*Object).publicLink != "" {
-			return o.(*Object).publicLink, nil
-		}
-
-		opts = rest.Opts{
-			Method:     "PUT",
-			Path:       "/files/" + o.(*Object).id,
-			Parameters: fieldsValue(),
-		}
-	}
-
-	shareLink := api.CreateSharedLink{}
-	var info api.Item
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &shareLink, &info)
-		return shouldRetry(resp, err)
-	})
-	return info.SharedLink.URL, err
-}
-
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {
@@ -955,7 +908,6 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	o.sha1 = info.SHA1
 	o.modTime = info.ModTime()
 	o.id = info.ID
-	o.publicLink = info.SharedLink.URL
 	return nil
 }
 
@@ -1135,7 +1087,6 @@ var (
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/box/upload.go

@@ -211,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		if reqSize >= int64(chunkSize) {
+			reqSize = int64(chunkSize)
 		}
 
 		// Make a block of memory

backend/cache/cache.go

@@ -6,13 +6,10 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math"
 	"os"
 	"os/signal"
 	"path"
 	"path/filepath"
-	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -24,7 +21,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fs/fspath"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/rc"
 	"github.com/ncw/rclone/fs/walk"
@@ -84,16 +80,8 @@ func init() {
 			Hide:     fs.OptionHideBoth,
 			Advanced: true,
 		}, {
-			Name:     "plex_insecure",
-			Help:     "Skip all certificate verifications when connecting to the Plex server",
-			Advanced: true,
-		}, {
-			Name: "chunk_size",
-			Help: `The size of a chunk (partial file data).
-
-Use lower numbers for slower connections. If the chunk size is
-changed, any downloaded chunks will be invalid and cache-chunk-path
-will need to be cleared or unexpected EOF errors will occur.`,
+			Name:    "chunk_size",
+			Help:    "The size of a chunk. Lower value good for slow connections but can affect seamless reading.",
 			Default: DefCacheChunkSize,
 			Examples: []fs.OptionExample{{
 				Value: "1m",
@@ -106,10 +94,8 @@ will need to be cleared or unexpected EOF errors will occur.`,
 				Help:  "10 MB",
 			}},
 		}, {
-			Name: "info_age",
-			Help: `How long to cache file structure information (directory listings, file size, times etc). 
-If all write operations are done through the cache then you can safely make
-this value very large as the cache store will also be updated in real time.`,
+			Name:    "info_age",
+			Help:    "How much time should object info (file size, file hashes etc) be stored in cache.\nUse a very high value if you don't plan on changing the source FS from outside the cache.\nAccepted units are: \"s\", \"m\", \"h\".",
 			Default: DefCacheInfoAge,
 			Examples: []fs.OptionExample{{
 				Value: "1h",
@@ -122,11 +108,8 @@ this value very large as the cache store will also be updated in real time.`,
 				Help:  "48 hours",
 			}},
 		}, {
-			Name: "chunk_total_size",
-			Help: `The total size that the chunks can take up on the local disk.
-
-If the cache exceeds this value then it will start to delete the
-oldest chunks until it goes under this value.`,
+			Name:    "chunk_total_size",
+			Help:    "The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.",
 			Default: DefCacheTotalChunkSize,
 			Examples: []fs.OptionExample{{
 				Value: "500M",
@@ -141,143 +124,63 @@ oldest chunks until it goes under this value.`,
 		}, {
 			Name:     "db_path",
 			Default:  filepath.Join(config.CacheDir, "cache-backend"),
-			Help:     "Directory to store file structure metadata DB.\nThe remote name is used as the DB file name.",
+			Help:     "Directory to cache DB",
 			Advanced: true,
 		}, {
-			Name:    "chunk_path",
-			Default: filepath.Join(config.CacheDir, "cache-backend"),
-			Help: `Directory to cache chunk files.
-
-Path to where partial file data (chunks) are stored locally. The remote
-name is appended to the final path.
-
-This config follows the "--cache-db-path". If you specify a custom
-location for "--cache-db-path" and don't specify one for "--cache-chunk-path"
-then "--cache-chunk-path" will use the same path as "--cache-db-path".`,
+			Name:     "chunk_path",
+			Default:  filepath.Join(config.CacheDir, "cache-backend"),
+			Help:     "Directory to cache chunk files",
 			Advanced: true,
 		}, {
 			Name:     "db_purge",
 			Default:  false,
-			Help:     "Clear all the cached data for this remote on start.",
+			Help:     "Purge the cache DB before",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
 		}, {
-			Name:    "chunk_clean_interval",
-			Default: DefCacheChunkCleanInterval,
-			Help: `How often should the cache perform cleanups of the chunk storage.
-The default value should be ok for most people. If you find that the
-cache goes over "cache-chunk-total-size" too often then try to lower
-this value to force it to perform cleanups more often.`,
+			Name:     "chunk_clean_interval",
+			Default:  DefCacheChunkCleanInterval,
+			Help:     "Interval at which chunk cleanup runs",
 			Advanced: true,
 		}, {
-			Name:    "read_retries",
-			Default: DefCacheReadRetries,
-			Help: `How many times to retry a read from a cache storage.
-
-Since reading from a cache stream is independent from downloading file
-data, readers can get to a point where there's no more data in the
-cache.  Most of the times this can indicate a connectivity issue if
-cache isn't able to provide file data anymore.
-
-For really slow connections, increase this to a point where the stream is
-able to provide data but your experience will be very stuttering.`,
+			Name:     "read_retries",
+			Default:  DefCacheReadRetries,
+			Help:     "How many times to retry a read from a cache storage",
 			Advanced: true,
 		}, {
-			Name:    "workers",
-			Default: DefCacheTotalWorkers,
-			Help: `How many workers should run in parallel to download chunks.
-
-Higher values will mean more parallel processing (better CPU needed)
-and more concurrent requests on the cloud provider.  This impacts
-several aspects like the cloud provider API limits, more stress on the
-hardware that rclone runs on but it also means that streams will be
-more fluid and data will be available much more faster to readers.
-
-**Note**: If the optional Plex integration is enabled then this
-setting will adapt to the type of reading performed and the value
-specified here will be used as a maximum number of workers to use.`,
+			Name:     "workers",
+			Default:  DefCacheTotalWorkers,
+			Help:     "How many workers should run in parallel to download chunks",
 			Advanced: true,
 		}, {
-			Name:    "chunk_no_memory",
-			Default: DefCacheChunkNoMemory,
-			Help: `Disable the in-memory cache for storing chunks during streaming.
-
-By default, cache will keep file data during streaming in RAM as well
-to provide it to readers as fast as possible.
-
-This transient data is evicted as soon as it is read and the number of
-chunks stored doesn't exceed the number of workers. However, depending
-on other settings like "cache-chunk-size" and "cache-workers" this footprint
-can increase if there are parallel streams too (multiple files being read
-at the same time).
-
-If the hardware permits it, use this feature to provide an overall better
-performance during streaming but it can also be disabled if RAM is not
-available on the local machine.`,
+			Name:     "chunk_no_memory",
+			Default:  DefCacheChunkNoMemory,
+			Help:     "Disable the in-memory cache for storing chunks during streaming",
 			Advanced: true,
 		}, {
-			Name:    "rps",
-			Default: int(DefCacheRps),
-			Help: `Limits the number of requests per second to the source FS (-1 to disable)
-
-This setting places a hard limit on the number of requests per second
-that cache will be doing to the cloud provider remote and try to
-respect that value by setting waits between reads.
-
-If you find that you're getting banned or limited on the cloud
-provider through cache and know that a smaller number of requests per
-second will allow you to work with it then you can use this setting
-for that.
-
-A good balance of all the other settings should make this setting
-useless but it is available to set for more special cases.
-
-**NOTE**: This will limit the number of requests during streams but
-other API calls to the cloud provider like directory listings will
-still pass.`,
+			Name:     "rps",
+			Default:  int(DefCacheRps),
+			Help:     "Limits the number of requests per second to the source FS. -1 disables the rate limiter",
 			Advanced: true,
 		}, {
-			Name:    "writes",
-			Default: DefCacheWrites,
-			Help: `Cache file data on writes through the FS
-
-If you need to read files immediately after you upload them through
-cache you can enable this flag to have their data stored in the
-cache store at the same time during upload.`,
+			Name:     "writes",
+			Default:  DefCacheWrites,
+			Help:     "Will cache file data on writes through the FS",
 			Advanced: true,
 		}, {
-			Name:    "tmp_upload_path",
-			Default: "",
-			Help: `Directory to keep temporary files until they are uploaded.
-
-This is the path where cache will use as a temporary storage for new
-files that need to be uploaded to the cloud provider.
-
-Specifying a value will enable this feature. Without it, it is
-completely disabled and files will be uploaded directly to the cloud
-provider`,
+			Name:     "tmp_upload_path",
+			Default:  "",
+			Help:     "Directory to keep temporary files until they are uploaded to the cloud storage",
 			Advanced: true,
 		}, {
-			Name:    "tmp_wait_time",
-			Default: DefCacheTmpWaitTime,
-			Help: `How long should files be stored in local cache before being uploaded
-
-This is the duration that a file must wait in the temporary location
-_cache-tmp-upload-path_ before it is selected for upload.
-
-Note that only one file is uploaded at a time and it can take longer
-to start the upload if a queue formed for this purpose.`,
+			Name:     "tmp_wait_time",
+			Default:  DefCacheTmpWaitTime,
+			Help:     "How long should files be stored in local cache before being uploaded",
 			Advanced: true,
 		}, {
-			Name:    "db_wait_time",
-			Default: DefCacheDbWaitTime,
-			Help: `How long to wait for the DB to be available - 0 is unlimited
-
-Only one process can have the DB open at any one time, so rclone waits
-for this duration for the DB to become available before it gives an
-error.
-
-If you set it to 0 then it will wait forever.`,
+			Name:     "db_wait_time",
+			Default:  DefCacheDbWaitTime,
+			Help:     "How long to wait for the DB to be available - 0 is unlimited",
 			Advanced: true,
 		}},
 	})
@@ -290,7 +193,6 @@ type Options struct {
 	PlexUsername       string        `config:"plex_username"`
 	PlexPassword       string        `config:"plex_password"`
 	PlexToken          string        `config:"plex_token"`
-	PlexInsecure       bool          `config:"plex_insecure"`
 	ChunkSize          fs.SizeSuffix `config:"chunk_size"`
 	InfoAge            fs.Duration   `config:"info_age"`
 	ChunkTotalSize     fs.SizeSuffix `config:"chunk_total_size"`
@@ -346,7 +248,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 	if opt.ChunkTotalSize < opt.ChunkSize*fs.SizeSuffix(opt.TotalWorkers) {
-		return nil, errors.Errorf("don't set cache-chunk-total-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)",
+		return nil, errors.Errorf("don't set cache-total-chunk-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)",
 			opt.ChunkTotalSize, opt.ChunkSize, opt.TotalWorkers)
 	}
 
@@ -359,15 +261,10 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath)
 	}
 
-	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(opt.Remote)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", opt.Remote)
-	}
-
-	remotePath := fspath.JoinRootPath(wPath, rootPath)
-	wrappedFs, wrapErr := wInfo.NewFs(wName, remotePath, wConfig)
+	remotePath := path.Join(opt.Remote, rpath)
+	wrappedFs, wrapErr := fs.NewFs(remotePath)
 	if wrapErr != nil && wrapErr != fs.ErrorIsFile {
-		return nil, errors.Wrapf(wrapErr, "failed to make remote %s:%s to wrap", wName, remotePath)
+		return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
 	}
 	var fsErr error
 	fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath)
@@ -393,7 +290,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	f.plexConnector = &plexConnector{}
 	if opt.PlexURL != "" {
 		if opt.PlexToken != "" {
-			f.plexConnector, err = newPlexConnectorWithToken(f, opt.PlexURL, opt.PlexToken, opt.PlexInsecure)
+			f.plexConnector, err = newPlexConnectorWithToken(f, opt.PlexURL, opt.PlexToken)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", opt.PlexURL)
 			}
@@ -403,7 +300,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 				if err != nil {
 					decPass = opt.PlexPassword
 				}
-				f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
+				f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, func(token string) {
 					m.Set("plex_token", token)
 				})
 				if err != nil {
@@ -471,7 +368,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	fs.Infof(name, "Chunk Clean Interval: %v", f.opt.ChunkCleanInterval)
 	fs.Infof(name, "Workers: %v", f.opt.TotalWorkers)
 	fs.Infof(name, "File Age: %v", f.opt.InfoAge)
-	if f.opt.StoreWrites {
+	if !f.opt.StoreWrites {
 		fs.Infof(name, "Cache Writes: enabled")
 	}
 
@@ -506,9 +403,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	}()
 
 	if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil {
-		pollInterval := make(chan time.Duration, 1)
-		pollInterval <- time.Duration(f.opt.ChunkCleanInterval)
-		doChangeNotify(f.receiveChangeNotify, pollInterval)
+		doChangeNotify(f.receiveChangeNotify, time.Duration(f.opt.ChunkCleanInterval))
 	}
 
 	f.features = (&fs.Features{
@@ -560,39 +455,6 @@ Eg
 		Title: "Get cache stats",
 		Help: `
 Show statistics for the cache remote.
-`,
-	})
-
-	rc.Add(rc.Call{
-		Path:  "cache/fetch",
-		Fn:    f.rcFetch,
-		Title: "Fetch file chunks",
-		Help: `
-Ensure the specified file chunks are cached on disk.
-
-The chunks= parameter specifies the file chunks to check.
-It takes a comma separated list of array slice indices.
-The slice indices are similar to Python slices: start[:end]
-
-start is the 0 based chunk number from the beginning of the file
-to fetch inclusive. end is 0 based chunk number from the beginning
-of the file to fetch exclusive.
-Both values can be negative, in which case they count from the back
-of the file. The value "-5:" represents the last 5 chunks of a file.
-
-Some valid examples are:
-":5,-5:" -> the first and last five chunks
-"0,-2" -> the first and the second last chunk
-"0:10" -> the first ten chunks
-
-Any parameter with a key that starts with "file" can be used to
-specify files to fetch, eg
-
-    rclone rc cache/fetch chunks=0 file=hello file2=home/goodbye
-
-File names will automatically be encrypted when the a crypt remote
-is used on top of the cache.
-
 `,
 	})
 
@@ -610,22 +472,6 @@ func (f *Fs) httpStats(in rc.Params) (out rc.Params, err error) {
 	return out, nil
 }
 
-func (f *Fs) unwrapRemote(remote string) string {
-	remote = cleanPath(remote)
-	if remote != "" {
-		// if it's wrapped by crypt we need to check what format we got
-		if cryptFs, yes := f.isWrappedByCrypt(); yes {
-			_, err := cryptFs.DecryptFileName(remote)
-			// if it failed to decrypt then it is a decrypted format and we need to encrypt it
-			if err != nil {
-				return cryptFs.EncryptFileName(remote)
-			}
-			// else it's an encrypted format and we can use it as it is
-		}
-	}
-	return remote
-}
-
 func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 	out = make(rc.Params)
 	remoteInt, ok := in["remote"]
@@ -639,9 +485,20 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 		withData = true
 	}
 
-	remote = f.unwrapRemote(remote)
-	if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
-		return out, errors.Errorf("%s doesn't exist in cache", remote)
+	if cleanPath(remote) != "" {
+		// if it's wrapped by crypt we need to check what format we got
+		if cryptFs, yes := f.isWrappedByCrypt(); yes {
+			_, err := cryptFs.DecryptFileName(remote)
+			// if it failed to decrypt then it is a decrypted format and we need to encrypt it
+			if err != nil {
+				remote = cryptFs.EncryptFileName(remote)
+			}
+			// else it's an encrypted format and we can use it as it is
+		}
+
+		if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
+			return out, errors.Errorf("%s doesn't exist in cache", remote)
+		}
 	}
 
 	co := NewObject(f, remote)
@@ -671,141 +528,6 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 	return out, nil
 }
 
-func (f *Fs) rcFetch(in rc.Params) (rc.Params, error) {
-	type chunkRange struct {
-		start, end int64
-	}
-	parseChunks := func(ranges string) (crs []chunkRange, err error) {
-		for _, part := range strings.Split(ranges, ",") {
-			var start, end int64 = 0, math.MaxInt64
-			switch ints := strings.Split(part, ":"); len(ints) {
-			case 1:
-				start, err = strconv.ParseInt(ints[0], 10, 64)
-				if err != nil {
-					return nil, errors.Errorf("invalid range: %q", part)
-				}
-				end = start + 1
-			case 2:
-				if ints[0] != "" {
-					start, err = strconv.ParseInt(ints[0], 10, 64)
-					if err != nil {
-						return nil, errors.Errorf("invalid range: %q", part)
-					}
-				}
-				if ints[1] != "" {
-					end, err = strconv.ParseInt(ints[1], 10, 64)
-					if err != nil {
-						return nil, errors.Errorf("invalid range: %q", part)
-					}
-				}
-			default:
-				return nil, errors.Errorf("invalid range: %q", part)
-			}
-			crs = append(crs, chunkRange{start: start, end: end})
-		}
-		return
-	}
-	walkChunkRange := func(cr chunkRange, size int64, cb func(chunk int64)) {
-		if size <= 0 {
-			return
-		}
-		chunks := (size-1)/f.ChunkSize() + 1
-
-		start, end := cr.start, cr.end
-		if start < 0 {
-			start += chunks
-		}
-		if end <= 0 {
-			end += chunks
-		}
-		if end <= start {
-			return
-		}
-		switch {
-		case start < 0:
-			start = 0
-		case start >= chunks:
-			return
-		}
-		switch {
-		case end <= start:
-			end = start + 1
-		case end >= chunks:
-			end = chunks
-		}
-		for i := start; i < end; i++ {
-			cb(i)
-		}
-	}
-	walkChunkRanges := func(crs []chunkRange, size int64, cb func(chunk int64)) {
-		for _, cr := range crs {
-			walkChunkRange(cr, size, cb)
-		}
-	}
-
-	v, ok := in["chunks"]
-	if !ok {
-		return nil, errors.New("missing chunks parameter")
-	}
-	s, ok := v.(string)
-	if !ok {
-		return nil, errors.New("invalid chunks parameter")
-	}
-	delete(in, "chunks")
-	crs, err := parseChunks(s)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid chunks parameter")
-	}
-	var files [][2]string
-	for k, v := range in {
-		if !strings.HasPrefix(k, "file") {
-			return nil, errors.Errorf("invalid parameter %s=%s", k, v)
-		}
-		switch v := v.(type) {
-		case string:
-			files = append(files, [2]string{v, f.unwrapRemote(v)})
-		default:
-			return nil, errors.Errorf("invalid parameter %s=%s", k, v)
-		}
-	}
-	type fileStatus struct {
-		Error         string
-		FetchedChunks int
-	}
-	fetchedChunks := make(map[string]fileStatus, len(files))
-	for _, pair := range files {
-		file, remote := pair[0], pair[1]
-		var status fileStatus
-		o, err := f.NewObject(remote)
-		if err != nil {
-			fetchedChunks[file] = fileStatus{Error: err.Error()}
-			continue
-		}
-		co := o.(*Object)
-		err = co.refreshFromSource(true)
-		if err != nil {
-			fetchedChunks[file] = fileStatus{Error: err.Error()}
-			continue
-		}
-		handle := NewObjectHandle(co, f)
-		handle.UseMemory = false
-		handle.scaleWorkers(1)
-		walkChunkRanges(crs, co.Size(), func(chunk int64) {
-			_, err := handle.getChunk(chunk * f.ChunkSize())
-			if err != nil {
-				if status.Error == "" {
-					status.Error = err.Error()
-				}
-			} else {
-				status.FetchedChunks++
-			}
-		})
-		fetchedChunks[file] = status
-	}
-
-	return rc.Params{"status": fetchedChunks}, nil
-}
-
 // receiveChangeNotify is a wrapper to notifications sent from the wrapped FS about changed files
 func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) {
 	if crypt, yes := f.isWrappedByCrypt(); yes {
@@ -870,18 +592,15 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 	}
 }
 
-// ChangeNotify can subscribe multiple callers
+// ChangeNotify can subsribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
-func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
+func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
 	f.parentsForgetMu.Lock()
 	defer f.parentsForgetMu.Unlock()
 	fs.Debugf(f, "subscribing to ChangeNotify")
 	f.parentsForgetFn = append(f.parentsForgetFn, notifyFunc)
-	go func() {
-		for range pollInterval {
-		}
-	}()
+	return make(chan bool)
 }
 
 // Name of the remote (as passed into NewFs)
@@ -982,6 +701,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		fs.Debugf(dir, "list: cached entries: %v", entries)
 		return entries, nil
 	}
+	// FIXME need to clean existing cached listing
 
 	// we first search any temporary files stored locally
 	var cachedEntries fs.DirEntries
@@ -1007,42 +727,27 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	}
 
 	// search from the source
-	sourceEntries, err := f.Fs.List(dir)
+	entries, err = f.Fs.List(dir)
 	if err != nil {
 		return nil, err
 	}
-	fs.Debugf(dir, "list: read %v from source", len(sourceEntries))
-	fs.Debugf(dir, "list: source entries: %v", sourceEntries)
-
-	sort.Sort(sourceEntries)
-	for _, entry := range entries {
-		entryRemote := entry.Remote()
-		i := sort.Search(len(sourceEntries), func(i int) bool { return sourceEntries[i].Remote() >= entryRemote })
-		if i < len(sourceEntries) && sourceEntries[i].Remote() == entryRemote {
-			continue
-		}
-		fp := path.Join(f.Root(), entryRemote)
-		switch entry.(type) {
-		case fs.Object:
-			_ = f.cache.RemoveObject(fp)
-		case fs.Directory:
-			_ = f.cache.RemoveDir(fp)
-		}
-		fs.Debugf(dir, "list: remove entry: %v", entryRemote)
-	}
-	entries = nil
+	fs.Debugf(dir, "list: read %v from source", len(entries))
+	fs.Debugf(dir, "list: source entries: %v", entries)
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
 	var batchDirectories []*Directory
-	sort.Sort(cachedEntries)
-	tmpCnt := len(cachedEntries)
-	for _, entry := range sourceEntries {
+	for _, entry := range entries {
 		switch o := entry.(type) {
 		case fs.Object:
 			// skip over temporary objects (might be uploading)
-			oRemote := o.Remote()
-			i := sort.Search(tmpCnt, func(i int) bool { return cachedEntries[i].Remote() >= oRemote })
-			if i < tmpCnt && cachedEntries[i].Remote() == oRemote {
+			found := false
+			for _, t := range cachedEntries {
+				if t.Remote() == o.Remote() {
+					found = true
+					break
+				}
+			}
+			if found {
 				continue
 			}
 			co := ObjectFromOriginal(f, o).persist()
@@ -1191,7 +896,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1287,7 +992,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1549,7 +1254,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantCopy
@@ -1625,7 +1330,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 
 	// if this is a temp object then we perform the changes locally
 	if srcObj.isTempFile() {
-		// we check if the feature is still active
+		// we check if the feature is stil active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantMove

backend/cache/cache_internal_test.go

@@ -4,9 +4,6 @@ package cache_test
 
 import (
 	"bytes"
-	"encoding/base64"
-	goflag "flag"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -15,12 +12,21 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"runtime/debug"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
+
+	"encoding/base64"
+	goflag "flag"
+	"fmt"
+	"runtime/debug"
+
+	"encoding/json"
+	"net/http"
+
 	"github.com/ncw/rclone/backend/cache"
 	"github.com/ncw/rclone/backend/crypt"
 	_ "github.com/ncw/rclone/backend/drive"
@@ -30,11 +36,10 @@ import (
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/object"
 	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/fs/rc/rcflags"
 	"github.com/ncw/rclone/fstest"
 	"github.com/ncw/rclone/vfs"
 	"github.com/ncw/rclone/vfs/vfsflags"
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -387,10 +392,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), testSize)
+	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -690,8 +695,8 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 }
 
 func TestInternalChangeSeenAfterRc(t *testing.T) {
-	cacheExpire := rc.Calls.Get("cache/expire")
-	assert.NotNil(t, cacheExpire)
+	rcflags.Opt.Enabled = true
+	rc.Start(&rcflags.Opt)
 
 	id := fmt.Sprintf("ticsarc%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
@@ -724,9 +729,13 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	require.NoError(t, err)
 	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
 
-	// Call the rc function
-	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
+	m := make(map[string]string)
+	res, err := http.Post(fmt.Sprintf("http://localhost:5572/cache/expire?remote=%s", "data.bin"), "application/json; charset=utf-8", strings.NewReader(""))
 	require.NoError(t, err)
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	_ = json.NewDecoder(res.Body).Decode(&m)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -736,21 +745,23 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
-	_, err = runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err := runInstance.list(t, rootFs, "")
 
 	// create some rand test data
 	testData2 := randStringBytes(int(chunkSize))
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
 
 	// list should have 1 item only
-	li1, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
+	li1, err = runInstance.list(t, rootFs, "")
 	require.Len(t, li1, 1)
 
-	// Call the rc function
-	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
+	m = make(map[string]string)
+	res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader(""))
 	require.NoError(t, err)
+	defer func() {
+		_ = res2.Body.Close()
+	}()
+	_ = json.NewDecoder(res2.Body).Decode(&m)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -758,7 +769,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// list should have 2 items now
 	li2, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
 	require.Len(t, li2, 2)
 }
 
@@ -1495,8 +1505,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 	var err error
 
 	if r.useMount {
-		var f *os.File
-		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
@@ -1506,8 +1515,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		}()
 		_, err = f.WriteString(data + append)
 	} else {
-		var obj1 fs.Object
-		obj1, err = rootFs.NewObject(src)
+		obj1, err := rootFs.NewObject(src)
 		if err != nil {
 			return err
 		}
@@ -1639,13 +1647,15 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	}
-	if f.Features().UnWrap != nil {
-		cfs, ok := f.Features().UnWrap().(*cache.Fs)
-		if ok {
-			return cfs, nil
+	} else {
+		if f.Features().UnWrap != nil {
+			cfs, ok := f.Features().UnWrap().(*cache.Fs)
+			if ok {
+				return cfs, nil
+			}
 		}
 	}
+
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/cache_test.go

@@ -15,9 +15,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   "TestCache:",
-		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		RemoteName: "TestCache:",
+		NilObject:  (*cache.Object)(nil),
 	})
 }

backend/cache/cache_upload_test.go

@@ -3,7 +3,6 @@
 package cache_test
 
 import (
-	"fmt"
 	"math/rand"
 	"os"
 	"path"
@@ -11,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"fmt"
+
 	"github.com/ncw/rclone/backend/cache"
 	_ "github.com/ncw/rclone/backend/drive"
 	"github.com/ncw/rclone/fs"

backend/cache/cache_upload_test.go.orig

@@ -0,0 +1,455 @@
+// +build !plan9
+
+package cache_test
+
+import (
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"fmt"
+
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalUploadTempDirCreated(t *testing.T) {
+	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
+	require.NoError(t, err)
+}
+
+func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
+	// create some rand test data
+	testSize := int64(524288000)
+	testReader := runInstance.randomReader(t, testSize)
+	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
+	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
+	// validate that it exists in temp fs
+	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.NoError(t, err)
+
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(524416032), ti.Size())
+	} else {
+		require.Equal(t, testSize, ti.Size())
+	}
+	de1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	runInstance.completeBackgroundUpload(t, "one", bu)
+	// check if it was removed from temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if it can be read
+	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
+	require.NoError(t, err)
+	require.Len(t, data2, 1024)
+}
+
+func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadMoveExistingFile(t *testing.T) {
+	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(10485760)
+	testReader := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	time.Sleep(time.Second * 5)
+	//_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	//require.NoError(t, err)
+
+	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
+	require.NoError(t, err)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadTempPathCleaned(t *testing.T) {
+	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(1048576)
+	testReader := runInstance.randomReader(t, testSize)
+	testReader2 := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2)
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second")))
+	require.False(t, os.IsNotExist(err))
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin")))
+	require.True(t, os.IsNotExist(err))
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadQueueMoreFiles(t *testing.T) {
+	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("test")
+	require.NoError(t, err)
+	minSize := 5242880
+	maxSize := 10485760
+	totalFiles := 10
+	rand.Seed(time.Now().Unix())
+
+	lastFile := ""
+	for i := 0; i < totalFiles; i++ {
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		testReader := runInstance.randomReader(t, size)
+		remote := "test/" + strconv.Itoa(i) + ".bin"
+		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
+
+		// validate that it exists in temp fs
+		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
+		require.NoError(t, err)
+		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
+
+		if runInstance.wrappedIsExternal && i < totalFiles-1 {
+			time.Sleep(time.Second * 3)
+		}
+		lastFile = remote
+	}
+
+	// check if cache lists all files, likely temp upload didn't finish yet
+	de1, err := runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+
+	// wait for background uploader to do its thing
+	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
+
+	// retry until we have no more temp files and fail if they don't go down to 0
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if cache lists all files
+	de1, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+}
+
+func TestInternalUploadTempFileOperations(t *testing.T) {
+	id := "tiutfo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove - allowed
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("second/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.NoError(t, err)
+		_, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+		require.Error(t, err)
+		var started bool
+		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
+		require.NoError(t, err)
+		require.False(t, started)
+		runInstance.mkdir(t, rootFs, "test")
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Rmdir - allowed
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "directory not empty")
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+	require.False(t, started)
+	require.NoError(t, err)
+
+	// test Move/Rename -- allowed
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.NoError(t, err)
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove -- allowed
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.NoError(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.Error(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.Error(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update -- allowed
+	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
+	require.NoError(t, err)
+	obj2, err := rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
+	require.Equal(t, "one content updated", string(data2))
+	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(67), tmpInfo.Size())
+	} else {
+		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	}
+
+	// test SetModTime -- allowed
+	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	require.NotEqual(t, secondModTime, firstModTime)
+	require.NotEqual(t, time.Time{}, firstModTime)
+	require.NotEqual(t, time.Time{}, secondModTime)
+}
+
+func TestInternalUploadUploadingFileOperations(t *testing.T) {
+	id := "tiuufo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.Error(t, err)
+	}
+
+	// test Rmdir
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test Move/Rename
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.Error(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.Error(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.Error(t, err)
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update - this seems to work. Why? FIXME
+	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
+	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
+	//	require.Equal(t, "one content", string(data2))
+	//
+	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	//	require.NoError(t, err)
+	//	if runInstance.rootIsCrypt {
+	//		require.Equal(t, int64(67), tmpInfo.Size())
+	//	} else {
+	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	//	}
+	//})
+	//require.Error(t, err)
+
+	// test SetModTime -- seems to work cause of previous
+	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//require.Equal(t, secondModTime, firstModTime)
+	//require.NotEqual(t, time.Time{}, firstModTime)
+	//require.NotEqual(t, time.Time{}, secondModTime)
+}

backend/cache/cache_upload_test.go.rej

@@ -0,0 +1,12 @@
+--- cache_upload_test.go
++++ cache_upload_test.go
+@@ -1500,9 +1469,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
+ 	}
+ 	r.tempFiles = nil
+ 	debug.FreeOSMemory()
+-	for k, v := range r.runDefaultFlagMap {
+-		_ = flag.Set(k, v)
+-	}
+ }
+ 
+ func (r *run) randomBytes(t *testing.T, size int64) []byte {

backend/cache/directory.go

@@ -3,9 +3,10 @@
 package cache
 
 import (
-	"path"
 	"time"
 
+	"path"
+
 	"github.com/ncw/rclone/fs"
 )
 

backend/cache/handle.go

@@ -5,11 +5,12 @@ package cache
 import (
 	"fmt"
 	"io"
+	"sync"
+	"time"
+
 	"path"
 	"runtime"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/operations"
@@ -48,13 +49,12 @@ type Handle struct {
 	offset         int64
 	seenOffsets    map[int64]bool
 	mu             sync.Mutex
-	workersWg      sync.WaitGroup
 	confirmReading chan bool
-	workers        int
-	maxWorkerID    int
-	UseMemory      bool
-	closed         bool
-	reading        bool
+
+	UseMemory bool
+	workers   []*worker
+	closed    bool
+	reading   bool
 }
 
 // NewObjectHandle returns a new Handle for an existing Object
@@ -95,7 +95,7 @@ func (r *Handle) String() string {
 
 // startReadWorkers will start the worker pool
 func (r *Handle) startReadWorkers() {
-	if r.workers > 0 {
+	if r.hasAtLeastOneWorker() {
 		return
 	}
 	totalWorkers := r.cacheFs().opt.TotalWorkers
@@ -117,27 +117,26 @@ func (r *Handle) startReadWorkers() {
 
 // scaleOutWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
-	current := r.workers
+	current := len(r.workers)
 	if current == desired {
 		return
 	}
 	if current > desired {
 		// scale in gracefully
-		for r.workers > desired {
+		for i := 0; i < current-desired; i++ {
 			r.preloadQueue <- -1
-			r.workers--
 		}
 	} else {
 		// scale out
-		for r.workers < desired {
+		for i := 0; i < desired-current; i++ {
 			w := &worker{
 				r:  r,
-				id: r.maxWorkerID,
+				ch: r.preloadQueue,
+				id: current + i,
 			}
-			r.workersWg.Add(1)
-			r.workers++
-			r.maxWorkerID++
 			go w.run()
+
+			r.workers = append(r.workers, w)
 		}
 	}
 	// ignore first scale out from 0
@@ -149,7 +148,7 @@ func (r *Handle) scaleWorkers(desired int) {
 func (r *Handle) confirmExternalReading() {
 	// if we have a max value of workers
 	// then we skip this step
-	if r.workers > 1 ||
+	if len(r.workers) > 1 ||
 		!r.cacheFs().plexConnector.isConfigured() {
 		return
 	}
@@ -179,7 +178,7 @@ func (r *Handle) queueOffset(offset int64) {
 			}
 		}
 
-		for i := 0; i < r.workers; i++ {
+		for i := 0; i < len(r.workers); i++ {
 			o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
 			if o < 0 || o >= r.cachedObject.Size() {
 				continue
@@ -194,6 +193,16 @@ func (r *Handle) queueOffset(offset int64) {
 	}
 }
 
+func (r *Handle) hasAtLeastOneWorker() bool {
+	oneWorker := false
+	for i := 0; i < len(r.workers); i++ {
+		if r.workers[i].isRunning() {
+			oneWorker = true
+		}
+	}
+	return oneWorker
+}
+
 // getChunk is called by the FS to retrieve a specific chunk of known start and size from where it can find it
 // it can be from transient or persistent cache
 // it will also build the chunk from the cache's specific chunk boundaries and build the final desired chunk in a buffer
@@ -234,7 +243,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	// not found in ram or
 	// the worker didn't managed to download the chunk in time so we abort and close the stream
 	if err != nil || len(data) == 0 || !found {
-		if r.workers == 0 {
+		if !r.hasAtLeastOneWorker() {
 			fs.Errorf(r, "out of workers")
 			return nil, io.ErrUnexpectedEOF
 		}
@@ -295,7 +304,14 @@ func (r *Handle) Close() error {
 	close(r.preloadQueue)
 	r.closed = true
 	// wait for workers to complete their jobs before returning
-	r.workersWg.Wait()
+	waitCount := 3
+	for i := 0; i < len(r.workers); i++ {
+		waitIdx := 0
+		for r.workers[i].isRunning() && waitIdx < waitCount {
+			time.Sleep(time.Second)
+			waitIdx++
+		}
+	}
 	r.memory.db.Flush()
 
 	fs.Debugf(r, "cache reader closed %v", r.offset)
@@ -332,9 +348,12 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 }
 
 type worker struct {
-	r  *Handle
-	rc io.ReadCloser
-	id int
+	r       *Handle
+	ch      <-chan int64
+	rc      io.ReadCloser
+	id      int
+	running bool
+	mu      sync.Mutex
 }
 
 // String is a representation of this worker
@@ -379,19 +398,33 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 	})
 }
 
+func (w *worker) isRunning() bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.running
+}
+
+func (w *worker) setRunning(f bool) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.running = f
+}
+
 // run is the main loop for the worker which receives offsets to preload
 func (w *worker) run() {
 	var err error
 	var data []byte
+	defer w.setRunning(false)
 	defer func() {
 		if w.rc != nil {
 			_ = w.rc.Close()
+			w.setRunning(false)
 		}
-		w.r.workersWg.Done()
 	}()
 
 	for {
-		chunkStart, open := <-w.r.preloadQueue
+		chunkStart, open := <-w.ch
+		w.setRunning(true)
 		if chunkStart < 0 || !open {
 			break
 		}

backend/cache/object.go

@@ -208,17 +208,11 @@ func (o *Object) SetModTime(t time.Time) error {
 
 // Open is used to request a specific part of the file using fs.RangeOption
 func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
-	var err error
-
-	if o.Object == nil {
-		err = o.refreshFromSource(true)
-	} else {
-		err = o.refresh()
-	}
-	if err != nil {
+	if err := o.refreshFromSource(true); err != nil {
 		return nil, err
 	}
 
+	var err error
 	cacheReader := NewObjectHandle(o, o.CacheFs)
 	var offset, limit int64 = 0, -1
 	for _, option := range options {

backend/cache/plex.go

@@ -3,19 +3,20 @@
 package cache
 
 import (
-	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
+	"sync"
+
+	"bytes"
+	"io/ioutil"
+
 	"github.com/ncw/rclone/fs"
-	cache "github.com/patrickmn/go-cache"
+	"github.com/patrickmn/go-cache"
 	"golang.org/x/net/websocket"
 )
 
@@ -53,7 +54,6 @@ type plexConnector struct {
 	username   string
 	password   string
 	token      string
-	insecure   bool
 	f          *Fs
 	mu         sync.Mutex
 	running    bool
@@ -63,7 +63,7 @@ type plexConnector struct {
 }
 
 // newPlexConnector connects to a Plex server and generates a token
-func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool, saveToken func(string)) (*plexConnector, error) {
+func newPlexConnector(f *Fs, plexURL, username, password string, saveToken func(string)) (*plexConnector, error) {
 	u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/"))
 	if err != nil {
 		return nil, err
@@ -75,7 +75,6 @@ func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool,
 		username:   username,
 		password:   password,
 		token:      "",
-		insecure:   insecure,
 		stateCache: cache.New(time.Hour, time.Minute),
 		saveToken:  saveToken,
 	}
@@ -84,7 +83,7 @@ func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool,
 }
 
 // newPlexConnector connects to a Plex server and generates a token
-func newPlexConnectorWithToken(f *Fs, plexURL, token string, insecure bool) (*plexConnector, error) {
+func newPlexConnectorWithToken(f *Fs, plexURL, token string) (*plexConnector, error) {
 	u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/"))
 	if err != nil {
 		return nil, err
@@ -94,7 +93,6 @@ func newPlexConnectorWithToken(f *Fs, plexURL, token string, insecure bool) (*pl
 		f:          f,
 		url:        u,
 		token:      token,
-		insecure:   insecure,
 		stateCache: cache.New(time.Hour, time.Minute),
 	}
 	pc.listenWebsocket()
@@ -109,26 +107,14 @@ func (p *plexConnector) closeWebsocket() {
 	p.running = false
 }
 
-func (p *plexConnector) websocketDial() (*websocket.Conn, error) {
-	u := strings.TrimRight(strings.Replace(strings.Replace(
-		p.url.String(), "http://", "ws://", 1), "https://", "wss://", 1), "/")
-	url := fmt.Sprintf(defPlexNotificationURL, u, p.token)
-
-	config, err := websocket.NewConfig(url, "http://localhost")
-	if err != nil {
-		return nil, err
-	}
-	if p.insecure {
-		config.TlsConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-	return websocket.DialConfig(config)
-}
-
 func (p *plexConnector) listenWebsocket() {
 	p.runningMu.Lock()
 	defer p.runningMu.Unlock()
 
-	conn, err := p.websocketDial()
+	u := strings.Replace(p.url.String(), "http://", "ws://", 1)
+	u = strings.Replace(u, "https://", "wss://", 1)
+	conn, err := websocket.Dial(fmt.Sprintf(defPlexNotificationURL, strings.TrimRight(u, "/"), p.token),
+		"", "http://localhost")
 	if err != nil {
 		fs.Errorf("plex", "%v", err)
 		return

backend/cache/storage_memory.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	cache "github.com/patrickmn/go-cache"
+	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 )
 

backend/cache/storage_persistent.go

@@ -3,17 +3,20 @@
 package cache
 
 import (
+	"time"
+
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
+
+	"io/ioutil"
+
+	"fmt"
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/ncw/rclone/fs"
@@ -398,7 +401,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), encoded)
+		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -809,7 +812,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), encoded)
+		err = bucket.Put([]byte(destPath), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -1023,7 +1026,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1049,7 +1052,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), encoded)
+			err = bucket.Put([]byte(destPath), []byte(encoded))
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}

backend/crypt/cipher.go

@@ -17,9 +17,11 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/pkg/errors"
-	"github.com/rfjakob/eme"
+
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/crypto/scrypt"
+
+	"github.com/rfjakob/eme"
 )
 
 // Constants
@@ -41,7 +43,6 @@ var (
 	ErrorBadDecryptControlChar   = errors.New("bad decryption - contains control chars")
 	ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize")
 	ErrorTooShortAfterDecode     = errors.New("too short after base32 decode")
-	ErrorTooLongAfterDecode      = errors.New("too long after base32 decode")
 	ErrorEncryptedFileTooShort   = errors.New("file is too short to be encrypted")
 	ErrorEncryptedFileBadHeader  = errors.New("file has truncated block header")
 	ErrorEncryptedBadMagic       = errors.New("not an encrypted file - bad magic string")
@@ -285,9 +286,6 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 		// not possible if decodeFilename() working correctly
 		return "", ErrorTooShortAfterDecode
 	}
-	if len(rawCiphertext) > 2048 {
-		return "", ErrorTooLongAfterDecode
-	}
 	paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt)
 	plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext)
 	if err != nil {
@@ -463,7 +461,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(newRune)
+			_, _ = result.WriteRune(rune(newRune))
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -748,7 +746,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retrieve the nonce
+	// retreive the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil

backend/crypt/cipher_test.go

@@ -194,10 +194,6 @@ func TestEncryptSegment(t *testing.T) {
 
 func TestDecryptSegment(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
-	longName := make([]byte, 3328)
-	for i := range longName {
-		longName[i] = 'a'
-	}
 	c, _ := newCipher(NameEncryptionStandard, "", "", true)
 	for _, test := range []struct {
 		in          string
@@ -205,7 +201,6 @@ func TestDecryptSegment(t *testing.T) {
 	}{
 		{"64=", ErrorBadBase32Encoding},
 		{"!", base32.CorruptInputError(0)},
-		{string(longName), ErrorTooLongAfterDecode},
 		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},

backend/crypt/crypt.go

@@ -4,6 +4,7 @@ package crypt
 import (
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fs/fspath"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/pkg/errors"
 )
@@ -67,16 +67,8 @@ func init() {
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
 		}, {
-			Name: "show_mapping",
-			Help: `For all files listed show how the names encrypt.
-
-If this flag is set then for each file that the remote is asked to
-list, it will log (at level INFO) a line stating the decrypted file
-name and the encrypted file name.
-
-This is so you can work out which encrypted names are which decrypted
-names just in case you need to do something with the encrypted file
-names, or for debugging purposes.`,
+			Name:     "show_mapping",
+			Help:     "For all files listed show how the names encrypt.",
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
@@ -122,7 +114,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -138,20 +130,16 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(remote, name+":") {
 		return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
 	}
-	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(remote)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
-	}
 	// Look for a file first
-	remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
-	wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)
+	remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
+	wrappedFs, err := fs.NewFs(remotePath)
 	// if that didn't produce a file, look for a directory
 	if err != fs.ErrorIsFile {
-		remotePath = fspath.JoinRootPath(wPath, cipher.EncryptDirName(rpath))
-		wrappedFs, err = wInfo.NewFs(wName, remotePath, wConfig)
+		remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
+		wrappedFs, err = fs.NewFs(remotePath)
 	}
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", wName, remotePath)
+		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
 	}
 	f := &Fs{
 		Fs:     wrappedFs,
@@ -169,10 +157,23 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
-		SetTier:                 true,
-		GetTier:                 true,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
+	doChangeNotify := wrappedFs.Features().ChangeNotify
+	if doChangeNotify != nil {
+		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
+			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+				decrypted, err := f.DecryptFileName(path)
+				if err != nil {
+					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+					return
+				}
+				notifyFunc(decrypted, entryType)
+			}
+			return doChangeNotify(wrappedNotifyFunc, pollInterval)
+		}
+	}
+
 	return f, err
 }
 
@@ -189,7 +190,6 @@ type Options struct {
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
-	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
@@ -532,16 +532,6 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
-// WrapFs returns the Fs that is wrapping this Fs
-func (f *Fs) WrapFs() fs.Fs {
-	return f.wrapper
-}
-
-// SetWrapper sets the Fs that is wrapping this Fs
-func (f *Fs) SetWrapper(wrapper fs.Fs) {
-	f.wrapper = wrapper
-}
-
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -553,7 +543,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 }
 
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calculates the hash given by HashType on the fly
+// src with it, and calcuates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
@@ -614,75 +604,6 @@ func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr
 	return m.Sums()[hashType], nil
 }
 
-// MergeDirs merges the contents of all the directories passed
-// in into the first one and rmdirs the other directories.
-func (f *Fs) MergeDirs(dirs []fs.Directory) error {
-	do := f.Fs.Features().MergeDirs
-	if do == nil {
-		return errors.New("MergeDirs not supported")
-	}
-	out := make([]fs.Directory, len(dirs))
-	for i, dir := range dirs {
-		out[i] = fs.NewDirCopy(dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
-	}
-	return do(out)
-}
-
-// DirCacheFlush resets the directory cache - used in testing
-// as an optional interface
-func (f *Fs) DirCacheFlush() {
-	do := f.Fs.Features().DirCacheFlush
-	if do != nil {
-		do()
-	}
-}
-
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(remote string) (string, error) {
-	do := f.Fs.Features().PublicLink
-	if do == nil {
-		return "", errors.New("PublicLink not supported")
-	}
-	o, err := f.NewObject(remote)
-	if err != nil {
-		// assume it is a directory
-		return do(f.cipher.EncryptDirName(remote))
-	}
-	return do(o.(*Object).Object.Remote())
-}
-
-// ChangeNotify calls the passed function with a path
-// that has had changes. If the implementation
-// uses polling, it should adhere to the given interval.
-func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
-	do := f.Fs.Features().ChangeNotify
-	if do == nil {
-		return
-	}
-	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		fs.Logf(f, "path %q entryType %d", path, entryType)
-		var (
-			err       error
-			decrypted string
-		)
-		switch entryType {
-		case fs.EntryDirectory:
-			decrypted, err = f.cipher.DecryptDirName(path)
-		case fs.EntryObject:
-			decrypted, err = f.cipher.DecryptFileName(path)
-		default:
-			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
-			return
-		}
-		if err != nil {
-			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-			return
-		}
-		notifyFunc(decrypted, entryType)
-	}
-	do(wrappedNotifyFunc, pollIntervalChan)
-}
-
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -841,34 +762,6 @@ func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
-// ID returns the ID of the Object if known, or "" if not
-func (o *Object) ID() string {
-	do, ok := o.Object.(fs.IDer)
-	if !ok {
-		return ""
-	}
-	return do.ID()
-}
-
-// SetTier performs changing storage tier of the Object if
-// multiple storage classes supported
-func (o *Object) SetTier(tier string) error {
-	do, ok := o.Object.(fs.SetTierer)
-	if !ok {
-		return errors.New("crypt: underlying remote does not support SetTier")
-	}
-	return do.SetTier(tier)
-}
-
-// GetTier returns storage tier or class of the Object
-func (o *Object) GetTier() string {
-	do, ok := o.Object.(fs.GetTierer)
-	if !ok {
-		return ""
-	}
-	return do.GetTier()
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -882,15 +775,7 @@ var (
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
-	_ fs.Wrapper         = (*Fs)(nil)
-	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirCacheFlusher = (*Fs)(nil)
-	_ fs.ChangeNotifier  = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_test.go

@@ -7,31 +7,13 @@ import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/crypt"
-	_ "github.com/ncw/rclone/backend/drive" // for integration tests
 	_ "github.com/ncw/rclone/backend/local"
-	_ "github.com/ncw/rclone/backend/swift" // for integration tests
 	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fstest"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	if *fstest.RemoteName == "" {
-		t.Skip("Skipping as -remote not set")
-	}
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   *fstest.RemoteName,
-		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableObjectMethods: []string{"MimeType"},
-	})
-}
-
 // TestStandard runs integration tests against the remote
 func TestStandard(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
 	name := "TestCrypt"
 	fstests.Run(t, &fstests.Opt{
@@ -43,15 +25,11 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
 // TestOff runs integration tests against the remote
 func TestOff(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-off")
 	name := "TestCrypt2"
 	fstests.Run(t, &fstests.Opt{
@@ -63,15 +41,11 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
 // TestObfuscate runs integration tests against the remote
 func TestObfuscate(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt3"
 	fstests.Run(t, &fstests.Opt{
@@ -83,7 +57,6 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableObjectMethods: []string{"MimeType"},
+		SkipBadWindowsCharacters: true,
 	})
 }

backend/drive/drive_internal_test.go

@@ -1,80 +1,62 @@
 package drive
 
 import (
-	"bytes"
 	"encoding/json"
-	"io"
-	"io/ioutil"
-	"mime"
-	"path/filepath"
-	"strings"
 	"testing"
 
-	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/operations"
-	"github.com/ncw/rclone/fstest/fstests"
+	"google.golang.org/api/drive/v3"
+
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/api/drive/v3"
 )
 
-func TestDriveScopes(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		want     []string
-		wantFlag bool
-	}{
-		{"", []string{
-			"https://www.googleapis.com/auth/drive",
-		}, false},
-		{" drive.file , drive.readonly", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.readonly",
-		}, false},
-		{" drive.file , drive.appfolder", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.appfolder",
-		}, true},
-	} {
-		got := driveScopes(test.in)
-		assert.Equal(t, test.want, got, test.in)
-		gotFlag := driveScopesContainsAppFolder(got)
-		assert.Equal(t, test.wantFlag, gotFlag, test.in)
-	}
-}
-
-/*
-var additionalMimeTypes = map[string]string{
-	"application/vnd.ms-excel.sheet.macroenabled.12":                          ".xlsm",
-	"application/vnd.ms-excel.template.macroenabled.12":                       ".xltm",
-	"application/vnd.ms-powerpoint.presentation.macroenabled.12":              ".pptm",
-	"application/vnd.ms-powerpoint.slideshow.macroenabled.12":                 ".ppsm",
-	"application/vnd.ms-powerpoint.template.macroenabled.12":                  ".potm",
-	"application/vnd.ms-powerpoint":                                           ".ppt",
-	"application/vnd.ms-word.document.macroenabled.12":                        ".docm",
-	"application/vnd.ms-word.template.macroenabled.12":                        ".dotm",
-	"application/vnd.openxmlformats-officedocument.presentationml.template":   ".potx",
-	"application/vnd.openxmlformats-officedocument.spreadsheetml.template":    ".xltx",
-	"application/vnd.openxmlformats-officedocument.wordprocessingml.template": ".dotx",
-	"application/vnd.sun.xml.writer":                                          ".sxw",
-	"text/richtext":                                                           ".rtf",
-}
-*/
+const exampleExportFormats = `{
+	"application/vnd.google-apps.document": [
+		"application/rtf",
+		"application/vnd.oasis.opendocument.text",
+		"text/html",
+		"application/pdf",
+		"application/epub+zip",
+		"application/zip",
+		"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+		"text/plain"
+	],
+	"application/vnd.google-apps.spreadsheet": [
+		"application/x-vnd.oasis.opendocument.spreadsheet",
+		"text/tab-separated-values",
+		"application/pdf",
+		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+		"text/csv",
+		"application/zip",
+		"application/vnd.oasis.opendocument.spreadsheet"
+	],
+	"application/vnd.google-apps.jam": [
+		"application/pdf"
+	],
+	"application/vnd.google-apps.script": [
+		"application/vnd.google-apps.script+json"
+	],
+	"application/vnd.google-apps.presentation": [
+		"application/vnd.oasis.opendocument.presentation",
+		"application/pdf",
+		"application/vnd.openxmlformats-officedocument.presentationml.presentation",
+		"text/plain"
+	],
+	"application/vnd.google-apps.form": [
+		"application/zip"
+	],
+	"application/vnd.google-apps.drawing": [
+		"image/svg+xml",
+		"image/png",
+		"application/pdf",
+		"image/jpeg"
+	]
+}`
 
 // Load the example export formats into exportFormats for testing
-func TestInternalLoadExampleFormats(t *testing.T) {
-	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
-	var about struct {
-		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
-		ImportFormats map[string][]string `json:"importFormats,omitempty"`
-	}
-	require.NoError(t, err)
-	require.NoError(t, json.Unmarshal(buf, &about))
-	_exportFormats = fixMimeTypeMap(about.ExportFormats)
-	_importFormats = fixMimeTypeMap(about.ImportFormats)
+func TestInternalLoadExampleExportFormats(t *testing.T) {
+	exportFormatsOnce.Do(func() {})
+	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats))
 }
 
 func TestInternalParseExtensions(t *testing.T) {
@@ -83,24 +65,27 @@ func TestInternalParseExtensions(t *testing.T) {
 		want    []string
 		wantErr error
 	}{
-		{"doc", []string{".doc"}, nil},
-		{" docx ,XLSX, 	pptx,svg", []string{".docx", ".xlsx", ".pptx", ".svg"}, nil},
-		{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
-		{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
+		{"doc", []string{"doc"}, nil},
+		{" docx ,XLSX, 	pptx,svg", []string{"docx", "xlsx", "pptx", "svg"}, nil},
+		{"docx,svg,Docx", []string{"docx", "svg"}, nil},
+		{"docx,potato,docx", []string{"docx"}, errors.New(`couldn't find mime type for extension "potato"`)},
 	} {
-		extensions, _, gotErr := parseExtensions(test.in)
+		f := new(Fs)
+		gotErr := f.parseExtensions(test.in)
 		if test.wantErr == nil {
 			assert.NoError(t, gotErr)
 		} else {
 			assert.EqualError(t, gotErr, test.wantErr.Error())
 		}
-		assert.Equal(t, test.want, extensions)
+		assert.Equal(t, test.want, f.extensions)
 	}
 
 	// Test it is appending
-	extensions, _, gotErr := parseExtensions("docx,svg", "docx,svg,xlsx")
-	assert.NoError(t, gotErr)
-	assert.Equal(t, []string{".docx", ".svg", ".xlsx"}, extensions)
+	f := new(Fs)
+	assert.Nil(t, f.parseExtensions("docx,svg"))
+	assert.Nil(t, f.parseExtensions("docx,svg,xlsx"))
+	assert.Equal(t, []string{"docx", "svg", "xlsx"}, f.extensions)
+
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
@@ -114,17 +99,17 @@ func TestInternalFindExportFormat(t *testing.T) {
 		wantMimeType  string
 	}{
 		{[]string{}, "", ""},
-		{[]string{".pdf"}, ".pdf", "application/pdf"},
-		{[]string{".pdf", ".rtf", ".xls"}, ".pdf", "application/pdf"},
-		{[]string{".xls", ".rtf", ".pdf"}, ".rtf", "application/rtf"},
-		{[]string{".xls", ".csv", ".svg"}, "", ""},
+		{[]string{"pdf"}, "pdf", "application/pdf"},
+		{[]string{"pdf", "rtf", "xls"}, "pdf", "application/pdf"},
+		{[]string{"xls", "rtf", "pdf"}, "rtf", "application/rtf"},
+		{[]string{"xls", "csv", "svg"}, "", ""},
 	} {
 		f := new(Fs)
-		f.exportExtensions = test.extensions
+		f.extensions = test.extensions
 		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
 		assert.Equal(t, test.wantExtension, gotExtension)
 		if test.wantExtension != "" {
-			assert.Equal(t, item.Name+gotExtension, gotFilename)
+			assert.Equal(t, item.Name+"."+gotExtension, gotFilename)
 		} else {
 			assert.Equal(t, "", gotFilename)
 		}
@@ -132,155 +117,3 @@ func TestInternalFindExportFormat(t *testing.T) {
 		assert.Equal(t, true, gotIsDocument)
 	}
 }
-
-func TestMimeTypesToExtension(t *testing.T) {
-	for mimeType, extension := range _mimeTypeToExtension {
-		extensions, err := mime.ExtensionsByType(mimeType)
-		assert.NoError(t, err)
-		assert.Contains(t, extensions, extension)
-	}
-}
-
-func TestExtensionToMimeType(t *testing.T) {
-	for mimeType, extension := range _mimeTypeToExtension {
-		gotMimeType := mime.TypeByExtension(extension)
-		mediatype, _, err := mime.ParseMediaType(gotMimeType)
-		assert.NoError(t, err)
-		assert.Equal(t, mimeType, mediatype)
-	}
-}
-
-func TestExtensionsForExportFormats(t *testing.T) {
-	if _exportFormats == nil {
-		t.Error("exportFormats == nil")
-	}
-	for fromMT, toMTs := range _exportFormats {
-		for _, toMT := range toMTs {
-			if !isInternalMimeType(toMT) {
-				extensions, err := mime.ExtensionsByType(toMT)
-				assert.NoError(t, err, "invalid MIME type %q", toMT)
-				assert.NotEmpty(t, extensions, "No extension found for %q (from: %q)", fromMT, toMT)
-			}
-		}
-	}
-}
-
-func TestExtensionsForImportFormats(t *testing.T) {
-	t.Skip()
-	if _importFormats == nil {
-		t.Error("_importFormats == nil")
-	}
-	for fromMT := range _importFormats {
-		if !isInternalMimeType(fromMT) {
-			extensions, err := mime.ExtensionsByType(fromMT)
-			assert.NoError(t, err, "invalid MIME type %q", fromMT)
-			assert.NotEmpty(t, extensions, "No extension found for %q", fromMT)
-		}
-	}
-}
-
-func (f *Fs) InternalTestDocumentImport(t *testing.T) {
-	oldAllow := f.opt.AllowImportNameChange
-	f.opt.AllowImportNameChange = true
-	defer func() {
-		f.opt.AllowImportNameChange = oldAllow
-	}()
-
-	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
-	require.NoError(t, err)
-
-	testFilesFs, err := fs.NewFs(testFilesPath)
-	require.NoError(t, err)
-
-	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
-	require.NoError(t, err)
-
-	err = operations.CopyFile(f, testFilesFs, "example2.doc", "example2.doc")
-	require.NoError(t, err)
-}
-
-func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
-	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
-	require.NoError(t, err)
-
-	testFilesFs, err := fs.NewFs(testFilesPath)
-	require.NoError(t, err)
-
-	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
-	require.NoError(t, err)
-
-	err = operations.CopyFile(f, testFilesFs, "example2.xlsx", "example1.ods")
-	require.NoError(t, err)
-}
-
-func (f *Fs) InternalTestDocumentExport(t *testing.T) {
-	var buf bytes.Buffer
-	var err error
-
-	f.exportExtensions, _, err = parseExtensions("txt")
-	require.NoError(t, err)
-
-	obj, err := f.NewObject("example2.txt")
-	require.NoError(t, err)
-
-	rc, err := obj.Open()
-	require.NoError(t, err)
-	defer func() { require.NoError(t, rc.Close()) }()
-
-	_, err = io.Copy(&buf, rc)
-	require.NoError(t, err)
-	text := buf.String()
-
-	for _, excerpt := range []string{
-		"Lorem ipsum dolor sit amet, consectetur",
-		"porta at ultrices in, consectetur at augue.",
-	} {
-		require.Contains(t, text, excerpt)
-	}
-}
-
-func (f *Fs) InternalTestDocumentLink(t *testing.T) {
-	var buf bytes.Buffer
-	var err error
-
-	f.exportExtensions, _, err = parseExtensions("link.html")
-	require.NoError(t, err)
-
-	obj, err := f.NewObject("example2.link.html")
-	require.NoError(t, err)
-
-	rc, err := obj.Open()
-	require.NoError(t, err)
-	defer func() { require.NoError(t, rc.Close()) }()
-
-	_, err = io.Copy(&buf, rc)
-	require.NoError(t, err)
-	text := buf.String()
-
-	require.True(t, strings.HasPrefix(text, "<html>"))
-	require.True(t, strings.HasSuffix(text, "</html>\n"))
-	for _, excerpt := range []string{
-		`<meta http-equiv="refresh"`,
-		`Loading <a href="`,
-	} {
-		require.Contains(t, text, excerpt)
-	}
-}
-
-func (f *Fs) InternalTest(t *testing.T) {
-	// These tests all depend on each other so run them as nested tests
-	t.Run("DocumentImport", func(t *testing.T) {
-		f.InternalTestDocumentImport(t)
-		t.Run("DocumentUpdate", func(t *testing.T) {
-			f.InternalTestDocumentUpdate(t)
-			t.Run("DocumentExport", func(t *testing.T) {
-				f.InternalTestDocumentExport(t)
-				t.Run("DocumentLink", func(t *testing.T) {
-					f.InternalTestDocumentLink(t)
-				})
-			})
-		})
-	})
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/drive/drive_test.go

@@ -1,11 +1,10 @@
 // Test Drive filesystem interface
-
-package drive
+package drive_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/drive"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -13,23 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestDrive:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize:  minChunkSize,
-			CeilChunkSize: fstests.NextPowerOfTwo,
-		},
+		NilObject:  (*drive.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/drive/test/about.json

@@ -1,178 +0,0 @@
-{
- "importFormats": {
-  "text/tab-separated-values": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/x-vnd.oasis.opendocument.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/jpeg": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/bmp": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/gif": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-excel.sheet.macroenabled.12": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.template": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-powerpoint.presentation.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-word.template.macroenabled.12": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/pjpeg": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.google-apps.script+text/plain": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/vnd.ms-excel": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.sun.xml.writer": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-word.document.macroenabled.12": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-powerpoint.slideshow.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "text/rtf": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/plain": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.oasis.opendocument.spreadsheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/x-vnd.oasis.opendocument.spreadsheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "image/png": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/x-vnd.oasis.opendocument.text": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/msword": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/pdf": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/json": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/x-msmetafile": [
-   "application/vnd.google-apps.drawing"
-  ],
-  "application/vnd.openxmlformats-officedocument.spreadsheetml.template": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.ms-powerpoint": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-excel.template.macroenabled.12": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "image/x-bmp": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/rtf": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.template": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/x-png": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/html": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.oasis.opendocument.text": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.google-apps.script+json": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.slideshow": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-powerpoint.template.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "text/csv": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.oasis.opendocument.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/jpg": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/richtext": [
-   "application/vnd.google-apps.document"
-  ]
- },
- "exportFormats": {
-  "application/vnd.google-apps.document": [
-   "application/rtf",
-   "application/vnd.oasis.opendocument.text",
-   "text/html",
-   "application/pdf",
-   "application/epub+zip",
-   "application/zip",
-   "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-   "text/plain"
-  ],
-  "application/vnd.google-apps.spreadsheet": [
-   "application/x-vnd.oasis.opendocument.spreadsheet",
-   "text/tab-separated-values",
-   "application/pdf",
-   "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-   "text/csv",
-   "application/zip",
-   "application/vnd.oasis.opendocument.spreadsheet"
-  ],
-  "application/vnd.google-apps.jam": [
-   "application/pdf"
-  ],
-  "application/vnd.google-apps.script": [
-   "application/vnd.google-apps.script+json"
-  ],
-  "application/vnd.google-apps.presentation": [
-   "application/vnd.oasis.opendocument.presentation",
-   "application/pdf",
-   "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-   "text/plain"
-  ],
-  "application/vnd.google-apps.form": [
-   "application/zip"
-  ],
-  "application/vnd.google-apps.drawing": [
-   "image/svg+xml",
-   "image/png",
-   "application/pdf",
-   "image/jpeg"
-  ]
- }
-}

backend/drive/upload.go

@@ -50,12 +50,11 @@ type resumableUpload struct {
 }
 
 // Upload the io.Reader in of size bytes with contentType and info
-func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
-	params := url.Values{
-		"alt":        {"json"},
-		"uploadType": {"resumable"},
-		"fields":     {partialFields},
-	}
+func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string, info *drive.File, remote string) (*drive.File, error) {
+	params := make(url.Values)
+	params.Set("alt", "json")
+	params.Set("uploadType", "resumable")
+	params.Set("fields", partialFields)
 	if f.isTeamDrive {
 		params.Set("supportsTeamDrives", "true")
 	}
@@ -183,7 +182,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionally on
+	// any other 2xx codes, so we parse it unconditionaly on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err

backend/dropbox/dropbox.go

@@ -31,11 +31,9 @@ import (
 	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
-	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/auth"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing"
-	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -81,8 +79,8 @@ const (
 	// Choose 48MB which is 91% of Maximum speed.  rclone by
 	// default does 4 transfers so this should use 4*48MB = 192MB
 	// by default.
-	defaultChunkSize = 48 * fs.MebiByte
-	maxChunkSize     = 150 * fs.MebiByte
+	defaultChunkSize = 48 * 1024 * 1024
+	maxChunkSize     = 150 * 1024 * 1024
 )
 
 var (
@@ -122,21 +120,9 @@ func init() {
 			Name: config.ConfigClientSecret,
 			Help: "Dropbox App Client Secret\nLeave blank normally.",
 		}, {
-			Name: "chunk_size",
-			Help: fmt.Sprintf(`Upload chunk size. (< %v).
-
-Any files larger than this will be uploaded in chunks of this size.
-
-Note that chunks are buffered in memory (one at a time) so rclone can
-deal with retries.  Setting this larger will increase the speed
-slightly (at most 10%% for 128MB in tests) at the cost of using more
-memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
-			Default:  defaultChunkSize,
-			Advanced: true,
-		}, {
-			Name:     "impersonate",
-			Help:     "Impersonate this user when using a business account.",
-			Default:  "",
+			Name:     "chunk_size",
+			Help:     fmt.Sprintf("Upload chunk size. Max %v.", fs.SizeSuffix(maxChunkSize)),
+			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
 		}},
 	})
@@ -144,8 +130,7 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize   fs.SizeSuffix `config:"chunk_size"`
-	Impersonate string        `config:"impersonate"`
+	ChunkSize fs.SizeSuffix `config:"chunk_size"`
 }
 
 // Fs represents a remote dropbox server
@@ -157,10 +142,9 @@ type Fs struct {
 	srv            files.Client   // the connection to the dropbox server
 	sharing        sharing.Client // as above, but for generating sharing links
 	users          users.Client   // as above, but for accessing user information
-	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *fs.Pacer      // To pace the API calls
+	pacer          *pacer.Pacer   // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 }
 
@@ -204,42 +188,14 @@ func shouldRetry(err error) (bool, error) {
 		return false, err
 	}
 	baseErrString := errors.Cause(err).Error()
-	// handle any official Retry-After header from Dropbox's SDK first
-	switch e := err.(type) {
-	case auth.RateLimitAPIError:
-		if e.RateLimitError.RetryAfter > 0 {
-			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
-			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
-		}
-		return true, err
-	}
-	// Keep old behavior for backward compatibility
-	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") || baseErrString == "" {
+	// FIXME there is probably a better way of doing this!
+	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
 		return true, err
 	}
 	return fserrors.ShouldRetry(err), err
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	const minChunkSize = fs.Byte
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	if cs > maxChunkSize {
-		return errors.Errorf("%s is greater than %s", cs, maxChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -247,9 +203,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "dropbox: chunk size")
+	if opt.ChunkSize > maxChunkSize {
+		return nil, errors.Errorf("chunk size too big, must be < %v", maxChunkSize)
 	}
 
 	// Convert the old token if it exists.  The old token was just
@@ -273,36 +228,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	config := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
 		Client:          oAuthClient,    // maybe???
 		HeaderGenerator: f.headerGenerator,
 	}
-
-	// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
-	f.team = team.New(config)
-
-	if opt.Impersonate != "" {
-
-		user := team.UserSelectorArg{
-			Email: opt.Impersonate,
-		}
-		user.Tag = "email"
-
-		members := []*team.UserSelectorArg{&user}
-		args := team.NewMembersGetInfoArgs(members)
-
-		memberIds, err := f.team.MembersGetInfo(args)
-
-		if err != nil {
-			return nil, errors.Wrapf(err, "invalid dropbox team member: %q", opt.Impersonate)
-		}
-
-		config.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
-	}
-
 	f.srv = files.New(config)
 	f.sharing = sharing.New(config)
 	f.users = users.New(config)

backend/dropbox/dropbox_test.go

@@ -1,10 +1,10 @@
 // Test Dropbox filesystem interface
-package dropbox
+package dropbox_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/dropbox"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -12,15 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestDropbox:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MaxChunkSize: maxChunkSize,
-		},
+		NilObject:  (*dropbox.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/ftp/ftp.go

@@ -15,7 +15,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -46,11 +45,6 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
-			}, {
-				Name:     "concurrency",
-				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-				Default:  0,
-				Advanced: true,
 			},
 		},
 	})
@@ -58,11 +52,10 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host        string `config:"host"`
-	User        string `config:"user"`
-	Pass        string `config:"pass"`
-	Port        string `config:"port"`
-	Concurrency int    `config:"concurrency"`
+	Host string `config:"host"`
+	User string `config:"user"`
+	Pass string `config:"pass"`
+	Port string `config:"port"`
 }
 
 // Fs represents a remote FTP server
@@ -77,7 +70,6 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
-	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -136,9 +128,6 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		f.tokens.Get()
-	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -158,9 +147,6 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		defer f.tokens.Put()
-	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -180,7 +166,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	f.poolMu.Unlock()
 }
 
-// NewFs constructs an Fs from the path, container:path
+// NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
@@ -212,7 +198,6 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
-		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -661,21 +646,7 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
-	var err error
-	errchan := make(chan error, 1)
-	go func() {
-		errchan <- f.rc.Close()
-	}()
-	// Wait for Close for up to 60 seconds
-	timer := time.NewTimer(60 * time.Second)
-	select {
-	case err = <-errchan:
-		timer.Stop()
-	case <-timer.C:
-		// if timer fired assume no error but connection dead
-		fs.Errorf(f.f, "Timeout when waiting for connection Close")
-		return nil
-	}
+	err := f.rc.Close()
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
@@ -733,11 +704,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	path := path.Join(o.fs.root, o.remote)
 	// remove the file if upload failed
 	remove := func() {
-		// Give the FTP server a chance to get its internal state in order after the error.
-		// The error may have been local in which case we closed the connection.  The server
-		// may still be dealing with it for a moment. A sleep isn't ideal but I haven't been
-		// able to think of a better method to find out if the server has finished - ncw
-		time.Sleep(1 * time.Second)
 		removeErr := o.Remove()
 		if removeErr != nil {
 			fs.Debugf(o, "Failed to remove: %v", removeErr)
@@ -751,7 +717,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	}
 	err = c.Stor(path, in)
 	if err != nil {
-		_ = c.Quit() // toss this connection to avoid sync errors
+		_ = c.Quit()
 		remove()
 		return errors.Wrap(err, "update stor")
 	}

backend/googlecloudstorage/googlecloudstorage.go

@@ -13,7 +13,6 @@ FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 erro
 */
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -43,8 +42,6 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
-
-	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
 )
 
@@ -144,22 +141,6 @@ func init() {
 				Value: "publicReadWrite",
 				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",
 			}},
-		}, {
-			Name: "bucket_policy_only",
-			Help: `Access checks should use bucket-level IAM policies.
-
-If you want to upload objects to a bucket with Bucket Policy Only set
-then you will need to set this.
-
-When it is set, rclone:
-
-- ignores ACLs set on buckets
-- ignores ACLs set on objects
-- creates buckets with Bucket Policy Only set
-
-Docs: https://cloud.google.com/storage/docs/bucket-policy-only
-`,
-			Default: false,
 		}, {
 			Name: "location",
 			Help: "Location for the newly created buckets.",
@@ -178,36 +159,21 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "asia-east1",
 				Help:  "Taiwan.",
-			}, {
-				Value: "asia-east2",
-				Help:  "Hong Kong.",
 			}, {
 				Value: "asia-northeast1",
 				Help:  "Tokyo.",
-			}, {
-				Value: "asia-south1",
-				Help:  "Mumbai.",
 			}, {
 				Value: "asia-southeast1",
 				Help:  "Singapore.",
 			}, {
 				Value: "australia-southeast1",
 				Help:  "Sydney.",
-			}, {
-				Value: "europe-north1",
-				Help:  "Finland.",
 			}, {
 				Value: "europe-west1",
 				Help:  "Belgium.",
 			}, {
 				Value: "europe-west2",
 				Help:  "London.",
-			}, {
-				Value: "europe-west3",
-				Help:  "Frankfurt.",
-			}, {
-				Value: "europe-west4",
-				Help:  "Netherlands.",
 			}, {
 				Value: "us-central1",
 				Help:  "Iowa.",
@@ -220,9 +186,6 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "us-west1",
 				Help:  "Oregon.",
-			}, {
-				Value: "us-west2",
-				Help:  "California.",
 			}},
 		}, {
 			Name: "storage_class",
@@ -257,7 +220,6 @@ type Options struct {
 	ServiceAccountCredentials string `config:"service_account_credentials"`
 	ObjectACL                 string `config:"object_acl"`
 	BucketACL                 string `config:"bucket_acl"`
-	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
 	Location                  string `config:"location"`
 	StorageClass              string `config:"storage_class"`
 }
@@ -273,7 +235,7 @@ type Fs struct {
 	bucket     string           // the bucket we are working on
 	bucketOKMu sync.Mutex       // mutex to protect bucket OK
 	bucketOK   bool             // true if we have created the bucket
-	pacer      *fs.Pacer        // To pace the API calls
+	pacer      *pacer.Pacer     // To pace the API calls
 }
 
 // Object describes a storage object
@@ -317,7 +279,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whether a given err rates being retried
+// shouldRetry determines whehter a given err rates being retried
 func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
@@ -365,7 +327,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 
@@ -383,7 +345,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// try loading service account credentials from env variable, then from a file
-	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
+	if opt.ServiceAccountCredentials != "" && opt.ServiceAccountFile != "" {
 		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, errors.Wrap(err, "error opening service account credentials file")
@@ -398,11 +360,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
 		if err != nil {
-			ctx := context.Background()
-			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
-			}
+			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
 		}
 	}
 
@@ -416,7 +374,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		opt:    *opt,
-		pacer:  fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -730,19 +688,8 @@ func (f *Fs) Mkdir(dir string) (err error) {
 		Location:     f.opt.Location,
 		StorageClass: f.opt.StorageClass,
 	}
-	if f.opt.BucketPolicyOnly {
-		bucket.IamConfiguration = &storage.BucketIamConfiguration{
-			BucketPolicyOnly: &storage.BucketIamConfigurationBucketPolicyOnly{
-				Enabled: true,
-			},
-		}
-	}
 	err = f.pacer.Call(func() (bool, error) {
-		insertBucket := f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket)
-		if !f.opt.BucketPolicyOnly {
-			insertBucket.PredefinedAcl(f.opt.BucketACL)
-		}
-		_, err = insertBucket.Do()
+		_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -1008,11 +955,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		insertObject := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
-		if !o.fs.opt.BucketPolicyOnly {
-			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
-		}
-		newObject, err = insertObject.Do()
+		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,5 +1,4 @@
 // Test GoogleCloudStorage filesystem interface
-
 package googlecloudstorage_test
 
 import (

backend/http/http.go

@@ -6,7 +6,6 @@ package http
 
 import (
 	"io"
-	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -41,26 +40,7 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "https://example.com",
 				Help:  "Connect to example.com",
-			}, {
-				Value: "https://user:pass@example.com",
-				Help:  "Connect to example.com using a username and password",
 			}},
-		}, {
-			Name: "no_slash",
-			Help: `Set this if the site doesn't end directories with /
-
-Use this if your target website does not use / on the end of
-directories.
-
-A / on the end of a path is how rclone normally tells the difference
-between files and directories.  If this flag is set, then rclone will
-treat all files with Content-Type: text/html as directories and read
-URLs from them rather than downloading them.
-
-Note that this may cause rclone to confuse genuine HTML files with
-directories.`,
-			Default:  false,
-			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -69,7 +49,6 @@ directories.`,
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
-	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -214,7 +193,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	}
 	err := o.stat()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "Stat failed")
 	}
 	return o, nil
 }
@@ -269,7 +248,7 @@ func parseName(base *url.URL, name string) (string, error) {
 	}
 	// calculate the name relative to the base
 	name = u.Path[len(base.Path):]
-	// mustn't be empty
+	// musn't be empty
 	if name == "" {
 		return "", errNameIsEmpty
 	}
@@ -288,20 +267,14 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var (
-		walk func(*html.Node)
-		seen = make(map[string]struct{})
-	)
+	var walk func(*html.Node)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						if _, found := seen[name]; !found {
-							names = append(names, name)
-							seen[name] = struct{}{}
-						}
+						names = append(names, name)
 					}
 					break
 				}
@@ -326,16 +299,14 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil {
-		defer fs.CheckClose(res.Body, &err)
-		if res.StatusCode == http.StatusNotFound {
-			return nil, fs.ErrorDirNotFound
-		}
+	if err == nil && res.StatusCode == http.StatusNotFound {
+		return nil, fs.ErrorDirNotFound
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
+	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -379,16 +350,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			switch err = file.stat(); err {
-			case nil:
-				entries = append(entries, file)
-			case fs.ErrorNotAFile:
-				// ...found a directory not a file
-				dir := fs.NewDir(remote, timeUnset)
-				entries = append(entries, dir)
-			default:
+			if err = file.stat(); err != nil {
 				fs.Debugf(remote, "skipping because of error: %v", err)
+				continue
 			}
+			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -450,9 +416,6 @@ func (o *Object) url() string {
 func (o *Object) stat() error {
 	url := o.url()
 	res, err := o.fs.httpClient.Head(url)
-	if err == nil && res.StatusCode == http.StatusNotFound {
-		return fs.ErrorObjectNotFound
-	}
 	err = statusError(res, err)
 	if err != nil {
 		return errors.Wrap(err, "failed to stat")
@@ -464,16 +427,6 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
-	// If NoSlash is set then check ContentType to see if it is a directory
-	if o.fs.opt.NoSlash {
-		mediaType, _, err := mime.ParseMediaType(o.contentType)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
-		}
-		if mediaType == "text/html" {
-			return fs.ErrorNotAFile
-		}
-	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -1,3 +1,5 @@
+// +build go1.8
+
 package http
 
 import (
@@ -63,7 +65,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
+func testListRoot(t *testing.T, f fs.Fs) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -91,29 +93,15 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	if noSlash {
-		assert.Equal(t, int64(-1), e.Size())
-		_, ok = e.(fs.Directory)
-		assert.True(t, ok)
-	} else {
-		assert.Equal(t, int64(41), e.Size())
-		_, ok = e.(*Object)
-		assert.True(t, ok)
-	}
+	assert.Equal(t, int64(7), e.Size())
+	_, ok = e.(*Object)
+	assert.True(t, ok)
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f, false)
-}
-
-func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
-	f.(*Fs).opt.NoSlash = true
-	defer tidy()
-
-	testListRoot(t, f, true)
+	testListRoot(t, f)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -156,11 +144,6 @@ func TestNewObject(t *testing.T) {
 
 	dt, ok := fstest.CheckTimeEqualWithPrecision(tObj, tFile, time.Second)
 	assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second))
-
-	// check object not found
-	o, err = f.NewObject("not found.txt")
-	assert.Nil(t, o)
-	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
 func TestOpen(t *testing.T) {
@@ -206,7 +189,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f, false)
+	testListRoot(t, f)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-<a href="two.html/file.txt">file.txt</a>
+potato

backend/hubic/hubic.go

@@ -9,10 +9,8 @@ package hubic
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/swift"
@@ -126,9 +124,7 @@ func (f *Fs) getCredentials() (err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
-		return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
+		return errors.Errorf("failed to get credentials: %s", resp.Status)
 	}
 	decoder := json.NewDecoder(resp.Body)
 	var result credentials

backend/hubic/hubic_test.go

@@ -11,9 +11,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:          "TestHubic:",
-		NilObject:           (*hubic.Object)(nil),
-		SkipFsCheckWrap:     true,
-		SkipObjectCheckWrap: true,
+		RemoteName: "TestHubic:",
+		NilObject:  (*hubic.Object)(nil),
 	})
 }

backend/jottacloud/api/types.go

@@ -9,10 +9,7 @@ import (
 )
 
 const (
-	// default time format for almost all request and responses
 	timeFormat = "2006-01-02-T15:04:05Z0700"
-	// the API server seems to use a different format
-	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
 )
 
 // Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -43,9 +40,6 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 // Return Time string in Jottacloud format
 func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 
-// APIString returns Time string in Jottacloud API format
-func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
-
 // Flag is a hacky type for checking if an attribute is present
 type Flag bool
 
@@ -64,15 +58,6 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	return attr, errors.New("unimplemented")
 }
 
-// TokenJSON is the struct representing the HTTP response from OAuth2
-// providers returning a token in JSON form.
-type TokenJSON struct {
-	AccessToken  string `json:"access_token"`
-	TokenType    string `json:"token_type"`
-	RefreshToken string `json:"refresh_token"`
-	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
-}
-
 /*
 GET http://www.jottacloud.com/JFS/<account>
 
@@ -248,17 +233,16 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
 
 // JottaFile represents a Jottacloud file
 type JottaFile struct {
-	XMLName         xml.Name
-	Name            string `xml:"name,attr"`
-	Deleted         Flag   `xml:"deleted,attr"`
-	PublicSharePath string `xml:"publicSharePath"`
-	State           string `xml:"currentRevision>state"`
-	CreatedAt       Time   `xml:"currentRevision>created"`
-	ModifiedAt      Time   `xml:"currentRevision>modified"`
-	Updated         Time   `xml:"currentRevision>updated"`
-	Size            int64  `xml:"currentRevision>size"`
-	MimeType        string `xml:"currentRevision>mime"`
-	MD5             string `xml:"currentRevision>md5"`
+	XMLName    xml.Name
+	Name       string `xml:"name,attr"`
+	Deleted    Flag   `xml:"deleted,attr"`
+	State      string `xml:"currentRevision>state"`
+	CreatedAt  Time   `xml:"currentRevision>created"`
+	ModifiedAt Time   `xml:"currentRevision>modified"`
+	Updated    Time   `xml:"currentRevision>updated"`
+	Size       int64  `xml:"currentRevision>size"`
+	MimeType   string `xml:"currentRevision>mime"`
+	MD5        string `xml:"currentRevision>md5"`
 }
 
 // Error is a custom Error for wrapping Jottacloud error responses
@@ -280,43 +264,3 @@ func (e *Error) Error() string {
 	}
 	return out
 }
-
-// AllocateFileRequest to prepare an upload to Jottacloud
-type AllocateFileRequest struct {
-	Bytes    int64  `json:"bytes"`
-	Created  string `json:"created"`
-	Md5      string `json:"md5"`
-	Modified string `json:"modified"`
-	Path     string `json:"path"`
-}
-
-// AllocateFileResponse for upload requests
-type AllocateFileResponse struct {
-	Name      string `json:"name"`
-	Path      string `json:"path"`
-	State     string `json:"state"`
-	UploadID  string `json:"upload_id"`
-	UploadURL string `json:"upload_url"`
-	Bytes     int64  `json:"bytes"`
-	ResumePos int64  `json:"resume_pos"`
-}
-
-// UploadResponse after an upload
-type UploadResponse struct {
-	Name      string      `json:"name"`
-	Path      string      `json:"path"`
-	Kind      string      `json:"kind"`
-	ContentID string      `json:"content_id"`
-	Bytes     int64       `json:"bytes"`
-	Md5       string      `json:"md5"`
-	Created   int64       `json:"created"`
-	Modified  int64       `json:"modified"`
-	Deleted   interface{} `json:"deleted"`
-	Mime      string      `json:"mime"`
-}
-
-// DeviceRegistrationResponse is the response to registering a device
-type DeviceRegistrationResponse struct {
-	ClientID     string `json:"client_id"`
-	ClientSecret string `json:"client_secret"`
-}

backend/jottacloud/jottacloud.go

@@ -7,8 +7,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -27,44 +25,21 @@ import (
 	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/fs/walk"
-	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
-	"golang.org/x/oauth2"
 )
 
 // Globals
 const (
-	minSleep                    = 10 * time.Millisecond
-	maxSleep                    = 2 * time.Second
-	decayConstant               = 2 // bigger for slower decay, exponential
-	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Sync" // nolint
-	rootURL                     = "https://www.jottacloud.com/jfs/"
-	apiURL                      = "https://api.jottacloud.com/files/v1/"
-	baseURL                     = "https://www.jottacloud.com/"
-	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
-	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
-	cachePrefix                 = "rclone-jcmd5-"
-	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
-	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
-	configUsername              = "user"
-	configClientID              = "client_id"
-	configClientSecret          = "client_secret"
-	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-)
-
-var (
-	// Description of how to auth for this app for a personal account
-	oauthConfig = &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
-		},
-		RedirectURL: oauthutil.RedirectLocalhostURL,
-	}
+	minSleep          = 10 * time.Millisecond
+	maxSleep          = 2 * time.Second
+	decayConstant     = 2 // bigger for slower decay, exponential
+	defaultDevice     = "Jotta"
+	defaultMountpoint = "Sync"
+	rootURL           = "https://www.jottacloud.com/jfs/"
+	apiURL            = "https://api.jottacloud.com"
+	cachePrefix       = "rclone-jcmd5-"
 )
 
 // Register with Fs
@@ -73,119 +48,13 @@ func init() {
 		Name:        "jottacloud",
 		Description: "JottaCloud",
 		NewFs:       NewFs,
-		Config: func(name string, m configmap.Mapper) {
-			tokenString, ok := m.Get("token")
-			if ok && tokenString != "" {
-				fmt.Printf("Already have a token - refresh?\n")
-				if !config.Confirm() {
-					return
-				}
-			}
-
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
-
-			// ask if we should create a device specifc token: https://github.com/ncw/rclone/issues/2995
-			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
-			if config.Confirm() {
-				// random generator to generate random device names
-				seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
-				randonDeviceNamePartLength := 21
-				randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
-				for i := range randomDeviceNamePart {
-					randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
-				}
-				randomDeviceName := "rclone-" + string(randomDeviceNamePart)
-				fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
-
-				values := url.Values{}
-				values.Set("device_id", randomDeviceName)
-
-				// all information comes from https://github.com/ttyridal/aiojotta/wiki/Jotta-protocol-3.-Authentication#token-authentication
-				opts := rest.Opts{
-					Method:       "POST",
-					RootURL:      registerURL,
-					ContentType:  "application/x-www-form-urlencoded",
-					ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
-					Parameters:   values,
-				}
-
-				var deviceRegistration api.DeviceRegistrationResponse
-				_, err := srv.CallJSON(&opts, nil, &deviceRegistration)
-				if err != nil {
-					log.Fatalf("Failed to register device: %v", err)
-				}
-
-				m.Set(configClientID, deviceRegistration.ClientID)
-				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
-				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
-			}
-
-			clientID, ok := m.Get(configClientID)
-			if !ok {
-				clientID = rcloneClientID
-			}
-			clientSecret, ok := m.Get(configClientSecret)
-			if !ok {
-				clientSecret = rcloneEncryptedClientSecret
-			}
-			oauthConfig.ClientID = clientID
-			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
-
-			username, ok := m.Get(configUsername)
-			if !ok {
-				log.Fatalf("No username defined")
-			}
-			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
-
-			// prepare out token request with username and password
-			values := url.Values{}
-			values.Set("grant_type", "PASSWORD")
-			values.Set("password", password)
-			values.Set("username", username)
-			values.Set("client_id", oauthConfig.ClientID)
-			values.Set("client_secret", oauthConfig.ClientSecret)
-			opts := rest.Opts{
-				Method:      "POST",
-				RootURL:     oauthConfig.Endpoint.AuthURL,
-				ContentType: "application/x-www-form-urlencoded",
-				Parameters:  values,
-			}
-
-			var jsonToken api.TokenJSON
-			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
-			if err != nil {
-				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
-				if resp != nil {
-					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
-						fmt.Printf("Enter verification code> ")
-						authCode := config.ReadLine()
-						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
-						opts.ExtraHeaders = make(map[string]string)
-						opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-						resp, err = srv.CallJSON(&opts, nil, &jsonToken)
-					}
-				}
-				if err != nil {
-					log.Fatalf("Failed to get resource token: %v", err)
-				}
-			}
-
-			var token oauth2.Token
-			token.AccessToken = jsonToken.AccessToken
-			token.RefreshToken = jsonToken.RefreshToken
-			token.TokenType = jsonToken.TokenType
-			token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
-
-			// finally save them in the config
-			err = oauthutil.PutToken(name, m, &token, true)
-			if err != nil {
-				log.Fatalf("Error while setting token: %s", err)
-			}
-		},
 		Options: []fs.Option{{
-			Name: configUsername,
-			Help: "User Name:",
+			Name: "user",
+			Help: "User Name",
+		}, {
+			Name:       "pass",
+			Help:       "Password.",
+			IsPassword: true,
 		}, {
 			Name:     "mountpoint",
 			Help:     "The mountpoint to use.",
@@ -202,21 +71,6 @@ func init() {
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
-		}, {
-			Name:     "hard_delete",
-			Help:     "Delete files permanently rather than putting them into the trash.",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     "unlink",
-			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     "upload_resume_limit",
-			Help:     "Files bigger than this can be resumed if the upload fail's.",
-			Default:  fs.SizeSuffix(10 * 1024 * 1024),
-			Advanced: true,
 		}},
 	})
 }
@@ -224,25 +78,21 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
+	Pass               string        `config:"pass"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
-	HardDelete         bool          `config:"hard_delete"`
-	Unlink             bool          `config:"unlink"`
-	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
 }
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name         string
-	root         string
-	user         string
-	opt          Options
-	features     *fs.Features
-	endpointURL  string
-	srv          *rest.Client
-	apiSrv       *rest.Client
-	pacer        *fs.Pacer
-	tokenRenewer *oauthutil.Renew // renew the token on expiry
+	name        string
+	root        string
+	user        string
+	opt         Options
+	features    *fs.Features
+	endpointURL string
+	srv         *rest.Client
+	pacer       *pacer.Pacer
 }
 
 // Object describes a jottacloud object
@@ -335,7 +185,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
 func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
-		Path:   urlPathEscape(f.user),
+		Path:   rest.URLPathEscape(f.user),
 	}
 
 	var resp *http.Response
@@ -356,7 +206,7 @@ func (f *Fs) setEndpointURL(mountpoint string) (err error) {
 	if err != nil {
 		return errors.Wrap(err, "failed to get endpoint url")
 	}
-	f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
+	f.endpointURL = rest.URLPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
 	return nil
 }
 
@@ -377,19 +227,9 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// Jottacloud want's '+' to be URL encoded even though the RFC states it's not reserved
-func urlPathEscape(in string) string {
-	return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
-}
-
-// filePathRaw returns an unescaped file path (f.root, file)
-func (f *Fs) filePathRaw(file string) string {
-	return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))
-}
-
 // filePath returns a escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
-	return urlPathEscape(f.filePathRaw(file))
+	return rest.URLPathEscape(path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file))))
 }
 
 // filePath returns a escaped file path (f.root, remote)
@@ -397,29 +237,6 @@ func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
 
-// Jottacloud requires the grant_type 'refresh_token' string
-// to be uppercase and throws a 400 Bad Request if we use the
-// lower case used by the oauth2 module
-//
-// This filter catches all refresh requests, reads the body,
-// changes the case and then sends it on
-func grantTypeFilter(req *http.Request) {
-	if tokenURL == req.URL.String() {
-		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
-		if err != nil {
-			return
-		}
-		_ = req.Body.Close()
-
-		// make the refresh token upper case
-		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
-
-		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
-	}
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -432,43 +249,25 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	clientID, ok := m.Get(configClientID)
-	if !ok {
-		clientID = rcloneClientID
-	}
-	clientSecret, ok := m.Get(configClientSecret)
-	if !ok {
-		clientSecret = rcloneEncryptedClientSecret
-	}
-	oauthConfig.ClientID = clientID
-	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+	user := config.FileGet(name, "user")
+	pass := config.FileGet(name, "pass")
 
-	// add jottacloud to the long list of sites that don't follow the oauth spec correctly
-	oauth2.RegisterBrokenAuthHeaderProvider("https://www.jottacloud.com/")
-
-	// the oauth client for the api servers needs
-	// a filter to fix the grant_type issues (see above)
-	baseClient := fshttp.NewClient(fs.Config)
-	if do, ok := baseClient.Transport.(interface {
-		SetRequestFilter(f func(req *http.Request))
-	}); ok {
-		do.SetRequestFilter(grantTypeFilter)
-	} else {
-		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
-	}
-	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
+	if opt.Pass != "" {
+		var err error
+		opt.Pass, err = obscure.Reveal(opt.Pass)
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't decrypt password")
+		}
 	}
 
 	f := &Fs{
-		name:   name,
-		root:   root,
-		user:   opt.User,
-		opt:    *opt,
-		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
-		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
-		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		name: name,
+		root: root,
+		user: opt.User,
+		opt:  *opt,
+		//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
+		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -476,13 +275,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 	}).Fill(f)
-	f.srv.SetErrorHandler(errorHandler)
 
-	// Renew the token in the background
-	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.readMetaDataForPath("")
-		return err
-	})
+	if user == "" || pass == "" {
+		return nil, errors.New("jottacloud needs user and password")
+	}
+
+	f.srv.SetUserPass(opt.User, opt.Pass)
+	f.srv.SetErrorHandler(errorHandler)
 
 	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
@@ -508,6 +307,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+
 	return f, nil
 }
 
@@ -524,7 +324,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData(false) // reads info and meta, returning an error
+		err = o.readMetaData() // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -572,7 +372,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	//fmt.Printf("List: %s\n", f.filePath(dir))
+	//fmt.Printf("List: %s\n", dir)
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   f.filePath(dir),
@@ -625,102 +425,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-// listFileDirFn is called from listFileDir to handle an object.
-type listFileDirFn func(fs.DirEntry) error
-
-// List the objects and directories into entries, from a
-// special kind of JottaFolder representing a FileDirLis
-func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, fn listFileDirFn) error {
-	pathPrefix := "/" + f.filePathRaw("") // Non-escaped prefix of API paths to be cut off, to be left with the remote path including the remoteStartPath
-	pathPrefixLength := len(pathPrefix)
-	startPath := path.Join(pathPrefix, remoteStartPath) // Non-escaped API path up to and including remoteStartPath, to decide if it should be created as a new dir object
-	startPathLength := len(startPath)
-	for i := range startFolder.Folders {
-		folder := &startFolder.Folders[i]
-		if folder.Deleted {
-			return nil
-		}
-		folderPath := restoreReservedChars(path.Join(folder.Path, folder.Name))
-		folderPathLength := len(folderPath)
-		var remoteDir string
-		if folderPathLength > pathPrefixLength {
-			remoteDir = folderPath[pathPrefixLength+1:]
-			if folderPathLength > startPathLength {
-				d := fs.NewDir(remoteDir, time.Time(folder.ModifiedAt))
-				err := fn(d)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		for i := range folder.Files {
-			file := &folder.Files[i]
-			if file.Deleted || file.State != "COMPLETED" {
-				continue
-			}
-			remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name))
-			o, err := f.newObjectWithInfo(remoteFile, file)
-			if err != nil {
-				return err
-			}
-			err = fn(o)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-// ListR lists the objects and directories of the Fs starting
-// from dir recursively into out.
-//
-// dir should be "" to start from the root, and should not
-// have trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-//
-// It should call callback for each tranche of entries read.
-// These need not be returned in any particular order.  If
-// callback returns an error then the listing will stop
-// immediately.
-//
-// Don't implement this unless you have a more efficient way
-// of listing recursively that doing a directory traversal.
-func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
-	opts := rest.Opts{
-		Method:     "GET",
-		Path:       f.filePath(dir),
-		Parameters: url.Values{},
-	}
-	opts.Parameters.Set("mode", "list")
-
-	var resp *http.Response
-	var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &result)
-		return shouldRetry(resp, err)
-	})
-	if err != nil {
-		if apiErr, ok := err.(*api.Error); ok {
-			// does not exist
-			if apiErr.StatusCode == http.StatusNotFound {
-				return fs.ErrorDirNotFound
-			}
-		}
-		return errors.Wrap(err, "couldn't list files")
-	}
-	list := walk.NewListRHelper(callback)
-	err = f.listFileDir(dir, &result, func(entry fs.DirEntry) error {
-		return list.Add(entry)
-	})
-	if err != nil {
-		return err
-	}
-	return list.Flush()
-}
-
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
@@ -794,11 +498,7 @@ func (f *Fs) purgeCheck(dir string, check bool) (err error) {
 		NoResponse: true,
 	}
 
-	if f.opt.HardDelete {
-		opts.Parameters.Set("rmDir", "true")
-	} else {
-		opts.Parameters.Set("dlDir", "true")
-	}
+	opts.Parameters.Set("dlDir", "true")
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -806,7 +506,7 @@ func (f *Fs) purgeCheck(dir string, check bool) (err error) {
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
-		return errors.Wrap(err, "couldn't purge directory")
+		return errors.Wrap(err, "rmdir failed")
 	}
 
 	// TODO: Parse response?
@@ -834,7 +534,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copies or moves directories or files depending on the method parameter
+// copyOrMoves copys or moves directories or files depending on the mthod parameter
 func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -852,6 +552,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
 	if err != nil {
 		return nil, err
 	}
+
 	return info, nil
 }
 
@@ -878,7 +579,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	info, err := f.copyOrMove("cp", srcObj.filePath(), remote)
 
 	if err != nil {
-		return nil, errors.Wrap(err, "couldn't copy file")
+		return nil, errors.Wrap(err, "copy failed")
 	}
 
 	return f.newObjectWithInfo(remote, info)
@@ -908,7 +609,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	info, err := f.copyOrMove("mv", srcObj.filePath(), remote)
 
 	if err != nil {
-		return nil, errors.Wrap(err, "couldn't move file")
+		return nil, errors.Wrap(err, "move failed")
 	}
 
 	return f.newObjectWithInfo(remote, info)
@@ -952,57 +653,11 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	_, err = f.copyOrMove("mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote)
 
 	if err != nil {
-		return errors.Wrap(err, "couldn't move directory")
+		return errors.Wrap(err, "moveDir failed")
 	}
 	return nil
 }
 
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(remote string) (link string, err error) {
-	opts := rest.Opts{
-		Method:     "GET",
-		Path:       f.filePath(remote),
-		Parameters: url.Values{},
-	}
-
-	if f.opt.Unlink {
-		opts.Parameters.Set("mode", "disableShare")
-	} else {
-		opts.Parameters.Set("mode", "enableShare")
-	}
-
-	var resp *http.Response
-	var result api.JottaFile
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &result)
-		return shouldRetry(resp, err)
-	})
-
-	if apiErr, ok := err.(*api.Error); ok {
-		// does not exist
-		if apiErr.StatusCode == http.StatusNotFound {
-			return "", fs.ErrorObjectNotFound
-		}
-	}
-	if err != nil {
-		if f.opt.Unlink {
-			return "", errors.Wrap(err, "couldn't remove public link")
-		}
-		return "", errors.Wrap(err, "couldn't create public link")
-	}
-	if f.opt.Unlink {
-		if result.PublicSharePath != "" {
-			return "", errors.Errorf("couldn't remove public link - %q", result.PublicSharePath)
-		}
-		return "", nil
-	}
-	if result.PublicSharePath == "" {
-		return "", errors.New("couldn't create public link - no link path received")
-	}
-	link = path.Join(baseURL, result.PublicSharePath)
-	return link, nil
-}
-
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
 	info, err := f.getAccountInfo()
@@ -1011,11 +666,8 @@ func (f *Fs) About() (*fs.Usage, error) {
 	}
 
 	usage := &fs.Usage{
-		Used: fs.NewUsageValue(info.Usage),
-	}
-	if info.Capacity > 0 {
-		usage.Total = fs.NewUsageValue(info.Capacity)
-		usage.Free = fs.NewUsageValue(info.Capacity - info.Usage)
+		Total: fs.NewUsageValue(info.Capacity),
+		Used:  fs.NewUsageValue(info.Usage),
 	}
 	return usage, nil
 }
@@ -1055,7 +707,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return 0
@@ -1071,24 +723,21 @@ func (o *Object) MimeType() string {
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.hasMetaData = true
-	o.size = info.Size
+	o.size = int64(info.Size)
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
 	o.modTime = time.Time(info.ModifiedAt)
 	return nil
 }
 
-func (o *Object) readMetaData(force bool) (err error) {
-	if o.hasMetaData && !force {
+func (o *Object) readMetaData() (err error) {
+	if o.hasMetaData {
 		return nil
 	}
 	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		return err
 	}
-	if info.Deleted {
-		return fs.ErrorObjectNotFound
-	}
 	return o.setMetaData(info)
 }
 
@@ -1097,7 +746,7 @@ func (o *Object) readMetaData(force bool) (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
-	err := o.readMetaData(false)
+	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1145,7 +794,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
 	// we need a MD5
 	md5Hasher := md5.New()
-	// use the teeReader to write to the local file AND calculate the MD5 while doing so
+	// use the teeReader to write to the local file AND caclulate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
 
 	// nothing to clean up by default
@@ -1218,74 +867,43 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		in = wrap(in)
 	}
 
-	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
+	var result api.JottaFile
 	opts := rest.Opts{
-		Method:       "POST",
-		Path:         "allocate",
-		ExtraHeaders: make(map[string]string),
-	}
-	fileDate := api.Time(src.ModTime()).APIString()
-
-	// the allocate request
-	var request = api.AllocateFileRequest{
-		Bytes:    size,
-		Created:  fileDate,
-		Modified: fileDate,
-		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
+		Method:        "POST",
+		Path:          o.filePath(),
+		Body:          in,
+		ContentType:   fs.MimeType(src),
+		ContentLength: &size,
+		ExtraHeaders:  make(map[string]string),
+		Parameters:    url.Values{},
 	}
 
-	// send it
-	var response api.AllocateFileResponse
+	opts.ExtraHeaders["JMd5"] = md5String
+	opts.Parameters.Set("cphash", md5String)
+	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
+	// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
+	opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
+
+	// Parameters observed in other implementations
+	//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
+	//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
+	//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
+	//opts.ExtraHeaders["jx_csid"] = ""
+	//opts.ExtraHeaders["jx_lisence"] = ""
+
+	opts.Parameters.Set("umode", "nomultipart")
+
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
+		resp, err = o.fs.srv.CallXML(&opts, nil, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
-	if response.State != "COMPLETED" {
-		// how much do we still have to upload?
-		remainingBytes := size - response.ResumePos
-		opts = rest.Opts{
-			Method:        "POST",
-			RootURL:       response.UploadURL,
-			ContentLength: &remainingBytes,
-			ContentType:   "application/octet-stream",
-			Body:          in,
-			ExtraHeaders:  make(map[string]string),
-		}
-		if response.ResumePos != 0 {
-			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
-		}
-
-		// copy the already uploaded bytes into the trash :)
-		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
-		if err != nil {
-			return err
-		}
-
-		// send the remaining bytes
-		resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
-		if err != nil {
-			return err
-		}
-
-		// finally update the meta data
-		o.hasMetaData = true
-		o.size = result.Bytes
-		o.md5 = result.Md5
-		o.modTime = time.Unix(result.Modified/1000, 0)
-	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
-		return o.readMetaData(true)
-	}
-
-	return nil
+	// TODO: Check returned Metadata? Timeout on big uploads?
+	return o.setMetaData(&result)
 }
 
 // Remove an object
@@ -1294,14 +912,9 @@ func (o *Object) Remove() error {
 		Method:     "POST",
 		Path:       o.filePath(),
 		Parameters: url.Values{},
-		NoResponse: true,
 	}
 
-	if o.fs.opt.HardDelete {
-		opts.Parameters.Set("rm", "true")
-	} else {
-		opts.Parameters.Set("dl", "true")
-	}
+	opts.Parameters.Set("dl", "true")
 
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallXML(&opts, nil, nil)
@@ -1311,14 +924,12 @@ func (o *Object) Remove() error {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs           = (*Fs)(nil)
-	_ fs.Purger       = (*Fs)(nil)
-	_ fs.Copier       = (*Fs)(nil)
-	_ fs.Mover        = (*Fs)(nil)
-	_ fs.DirMover     = (*Fs)(nil)
-	_ fs.ListRer      = (*Fs)(nil)
-	_ fs.PublicLinker = (*Fs)(nil)
-	_ fs.Abouter      = (*Fs)(nil)
-	_ fs.Object       = (*Object)(nil)
-	_ fs.MimeTyper    = (*Object)(nil)
+	_ fs.Fs        = (*Fs)(nil)
+	_ fs.Purger    = (*Fs)(nil)
+	_ fs.Copier    = (*Fs)(nil)
+	_ fs.Mover     = (*Fs)(nil)
+	_ fs.DirMover  = (*Fs)(nil)
+	_ fs.Abouter   = (*Fs)(nil)
+	_ fs.Object    = (*Object)(nil)
+	_ fs.MimeTyper = (*Object)(nil)
 )

backend/jottacloud/jottacloud_internal_test.go

@@ -4,25 +4,50 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"testing"
 
-	"github.com/ncw/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// A test reader to return a test pattern of size
+type testReader struct {
+	size int64
+	c    byte
+}
+
+// Reader is the interface that wraps the basic Read method.
+func (r *testReader) Read(p []byte) (n int, err error) {
+	for i := range p {
+		if r.size <= 0 {
+			return n, io.EOF
+		}
+		p[i] = r.c
+		r.c = (r.c + 1) % 253
+		r.size--
+		n++
+	}
+	return
+}
+
 func TestReadMD5(t *testing.T) {
+	// smoke test the reader
+	b, err := ioutil.ReadAll(&testReader{size: 10})
+	require.NoError(t, err)
+	assert.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, b)
+
 	// Check readMD5 for different size and threshold
 	for _, size := range []int64{0, 1024, 10 * 1024, 100 * 1024} {
 		t.Run(fmt.Sprintf("%d", size), func(t *testing.T) {
 			hasher := md5.New()
-			n, err := io.Copy(hasher, readers.NewPatternReader(size))
+			n, err := io.Copy(hasher, &testReader{size: size})
 			require.NoError(t, err)
 			assert.Equal(t, n, size)
 			wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
 			for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
 				t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
-					in := readers.NewPatternReader(size)
+					in := &testReader{size: size}
 					gotMD5, out, cleanup, err := readMD5(in, size, threshold)
 					defer cleanup()
 					require.NoError(t, err)

backend/jottacloud/replace.go

@@ -2,7 +2,7 @@
 Translate file names for JottaCloud adapted from OneDrive
 
 
-The following characters are JottaCloud reserved characters, and can't
+The following characters are JottaClous reserved characters, and can't
 be used in JottaCloud folder and file names.
 
   jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"
@@ -27,14 +27,21 @@ import (
 var (
 	charMap = map[rune]rune{
 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+		'+':  '＋', // FULLWIDTH PLUS SIGN
 		'*':  '＊', // FULLWIDTH ASTERISK
 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
 		'?':  '？', // FULLWIDTH QUESTION MARK
+		'!':  '！', // FULLWIDTH EXCLAMATION MARK
+		'&':  '＆', // FULLWIDTH AMPERSAND
 		':':  '：', // FULLWIDTH COLON
 		';':  '；', // FULLWIDTH SEMICOLON
 		'|':  '｜', // FULLWIDTH VERTICAL LINE
+		'#':  '＃', // FULLWIDTH NUMBER SIGN
+		'%':  '％', // FULLWIDTH PERCENT SIGN
 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
+		'\'': '＇', // FULLWIDTH APOSTROPHE
+		'~':  '～', // FULLWIDTH TILDE
 		' ':  '␠', // SYMBOL FOR SPACE
 	}
 	invCharMap           map[rune]rune

backend/jottacloud/replace_test.go

@@ -9,8 +9,8 @@ func TestReplace(t *testing.T) {
 	}{
 		{"", ""},
 		{"abc 123", "abc 123"},
-		{`\*<>?:;|"`, `＼＊＜＞？：；｜＂`},
-		{`\*<>?:;|"\*<>?:;|"`, `＼＊＜＞？：；｜＂＼＊＜＞？：；｜＂`},
+		{`\+*<>?!&:;|#%"'~`, `＼＋＊＜＞？！＆：；｜＃％＂＇～`},
+		{`\+*<>?!&:;|#%"'~\+*<>?!&:;|#%"'~`, `＼＋＊＜＞？！＆：；｜＃％＂＇～＼＋＊＜＞？！＆：；｜＃％＂＇～`},
 		{" leading space", "␠leading space"},
 		{"trailing space ", "trailing space␠"},
 		{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},

backend/koofr/koofr.go

@@ -1,589 +0,0 @@
-package koofr
-
-import (
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
-	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fs/hash"
-
-	httpclient "github.com/koofr/go-httpclient"
-	koofrclient "github.com/koofr/go-koofrclient"
-)
-
-// Register Fs with rclone
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "koofr",
-		Description: "Koofr",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "The Koofr API endpoint to use",
-				Default:  "https://app.koofr.net",
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "mountid",
-				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
-				Required: false,
-				Default:  "",
-				Advanced: true,
-			}, {
-				Name:     "user",
-				Help:     "Your Koofr user name",
-				Required: true,
-			}, {
-				Name:       "password",
-				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
-				IsPassword: true,
-				Required:   true,
-			},
-		},
-	})
-}
-
-// Options represent the configuration of the Koofr backend
-type Options struct {
-	Endpoint string `config:"endpoint"`
-	MountID  string `config:"mountid"`
-	User     string `config:"user"`
-	Password string `config:"password"`
-}
-
-// A Fs is a representation of a remote Koofr Fs
-type Fs struct {
-	name     string
-	mountID  string
-	root     string
-	opt      Options
-	features *fs.Features
-	client   *koofrclient.KoofrClient
-}
-
-// An Object on the remote Koofr Fs
-type Object struct {
-	fs     *Fs
-	remote string
-	info   koofrclient.FileInfo
-}
-
-func base(pth string) string {
-	rv := path.Base(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-func dir(pth string) string {
-	rv := path.Dir(pth)
-	if rv == "" || rv == "." {
-		rv = "/"
-	}
-	return rv
-}
-
-// String returns a string representation of the remote Object
-func (o *Object) String() string {
-	return o.remote
-}
-
-// Remote returns the remote path of the Object, relative to Fs root
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification time of the Object
-func (o *Object) ModTime() time.Time {
-	return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000)
-}
-
-// Size return the size of the Object in bytes
-func (o *Object) Size() int64 {
-	return o.info.Size
-}
-
-// Fs returns a reference to the Koofr Fs containing the Object
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns an MD5 hash of the Object
-func (o *Object) Hash(typ hash.Type) (string, error) {
-	if typ == hash.MD5 {
-		return o.info.Hash, nil
-	}
-	return "", nil
-}
-
-// fullPath returns full path of the remote Object (including Fs root)
-func (o *Object) fullPath() string {
-	return o.fs.fullPath(o.remote)
-}
-
-// Storable returns true if the Object is storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime is not supported
-func (o *Object) SetModTime(mtime time.Time) error {
-	return nil
-}
-
-// Open opens the Object for reading
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
-	var sOff, eOff int64 = 0, -1
-
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.SeekOption:
-			sOff = x.Offset
-		case *fs.RangeOption:
-			sOff = x.Start
-			eOff = x.End
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if sOff == 0 && eOff < 0 {
-		return o.fs.client.FilesGet(o.fs.mountID, o.fullPath())
-	}
-	if sOff < 0 {
-		sOff = o.Size() - eOff
-		eOff = o.Size()
-	}
-	if eOff > o.Size() {
-		eOff = o.Size()
-	}
-	span := &koofrclient.FileSpan{
-		Start: sOff,
-		End:   eOff,
-	}
-	return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span)
-}
-
-// Update updates the Object contents
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := o.fullPath()
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err := o.fs.mkdir(dirPath)
-	if err != nil {
-		return err
-	}
-	info, err := o.fs.client.FilesPutOptions(o.fs.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return err
-	}
-	o.info = *info
-	return nil
-}
-
-// Remove deletes the remote Object
-func (o *Object) Remove() error {
-	return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath())
-}
-
-// Name returns the name of the Fs
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root returns the root path of the Fs
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a string representation of the Fs
-func (f *Fs) String() string {
-	return "koofr:" + f.mountID + ":" + f.root
-}
-
-// Features returns the optional features supported by this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Precision denotes that setting modification times is not supported
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns a set of hashes are Provided by the Fs
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-}
-
-// fullPath constructs a full, absolute path from a Fs root relative path,
-func (f *Fs) fullPath(part string) string {
-	return path.Join("/", f.root, part)
-}
-
-// NewFs constructs a new filesystem given a root path and configuration options
-func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
-	opt := new(Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-	pass, err := obscure.Reveal(opt.Password)
-	if err != nil {
-		return nil, err
-	}
-	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
-	basicAuth := fmt.Sprintf("Basic %s",
-		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
-	client.HTTPClient.Headers.Set("Authorization", basicAuth)
-	mounts, err := client.Mounts()
-	if err != nil {
-		return nil, err
-	}
-	f := &Fs{
-		name:   name,
-		root:   root,
-		opt:    *opt,
-		client: client,
-	}
-	f.features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          false,
-		BucketBased:             false,
-		CanHaveEmptyDirectories: true,
-	}).Fill(f)
-	for _, m := range mounts {
-		if opt.MountID != "" {
-			if m.Id == opt.MountID {
-				f.mountID = m.Id
-				break
-			}
-		} else if m.IsPrimary {
-			f.mountID = m.Id
-			break
-		}
-	}
-	if f.mountID == "" {
-		if opt.MountID == "" {
-			return nil, errors.New("Failed to find primary mount")
-		}
-		return nil, errors.New("Failed to find mount " + opt.MountID)
-	}
-	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
-	if err == nil && rootFile.Type != "dir" {
-		f.root = dir(f.root)
-		err = fs.ErrorIsFile
-	} else {
-		err = nil
-	}
-	return f, err
-}
-
-// List returns a list of items in a directory
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return nil, translateErrorsDir(err)
-	}
-	entries = make([]fs.DirEntry, len(files))
-	for i, file := range files {
-		if file.Type == "dir" {
-			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
-		} else {
-			entries[i] = &Object{
-				fs:     f,
-				info:   file,
-				remote: path.Join(dir, file.Name),
-			}
-		}
-	}
-	return entries, nil
-}
-
-// NewObject creates a new remote Object for a given remote path
-func (f *Fs) NewObject(remote string) (obj fs.Object, err error) {
-	info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote))
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	if info.Type == "dir" {
-		return nil, fs.ErrorNotAFile
-	}
-	return &Object{
-		fs:     f,
-		info:   info,
-		remote: remote,
-	}, nil
-}
-
-// Put updates a remote Object
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
-	putopts := &koofrclient.PutFilter{
-		ForceOverwrite:    true,
-		NoRename:          true,
-		IgnoreNonExisting: true,
-	}
-	fullPath := f.fullPath(src.Remote())
-	dirPath := dir(fullPath)
-	name := base(fullPath)
-	err = f.mkdir(dirPath)
-	if err != nil {
-		return nil, err
-	}
-	info, err := f.client.FilesPutOptions(f.mountID, dirPath, name, in, putopts)
-	if err != nil {
-		return nil, translateErrorsObject(err)
-	}
-	return &Object{
-		fs:     f,
-		info:   *info,
-		remote: src.Remote(),
-	}, nil
-}
-
-// PutStream updates a remote Object with a stream of unknown size
-func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(in, src, options...)
-}
-
-// isBadRequest is a predicate which holds true iff the error returned was
-// HTTP status 400
-func isBadRequest(err error) bool {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusBadRequest {
-			return true
-		}
-	}
-	return false
-}
-
-// translateErrorsDir translates koofr errors to rclone errors (for a dir
-// operation)
-func translateErrorsDir(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorDirNotFound
-		}
-	}
-	return err
-}
-
-// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
-func translateErrorsObject(err error) error {
-	switch err := err.(type) {
-	case httpclient.InvalidStatusError:
-		if err.Got == http.StatusNotFound {
-			return fs.ErrorObjectNotFound
-		}
-	}
-	return err
-}
-
-// mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
-func (f *Fs) mkdir(fullPath string) error {
-	if fullPath == "/" {
-		return nil
-	}
-	info, err := f.client.FilesInfo(f.mountID, fullPath)
-	if err == nil && info.Type == "dir" {
-		return nil
-	}
-	err = translateErrorsDir(err)
-	if err != nil && err != fs.ErrorDirNotFound {
-		return err
-	}
-	dirs := strings.Split(fullPath, "/")
-	parent := "/"
-	for _, part := range dirs {
-		if part == "" {
-			continue
-		}
-		info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part))
-		if err != nil || info.Type != "dir" {
-			err = translateErrorsDir(err)
-			if err != nil && err != fs.ErrorDirNotFound {
-				return err
-			}
-			err = f.client.FilesNewFolder(f.mountID, parent, part)
-			if err != nil && !isBadRequest(err) {
-				return err
-			}
-		}
-		parent = path.Join(parent, part)
-	}
-	return nil
-}
-
-// Mkdir creates a directory at the given remote path. Creates ancestors if
-// necessary
-func (f *Fs) Mkdir(dir string) error {
-	fullPath := f.fullPath(dir)
-	return f.mkdir(fullPath)
-}
-
-// Rmdir removes an (empty) directory at the given remote path
-func (f *Fs) Rmdir(dir string) error {
-	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	if len(files) > 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-	err = f.client.FilesDelete(f.mountID, f.fullPath(dir))
-	if err != nil {
-		return translateErrorsDir(err)
-	}
-	return nil
-}
-
-// Copy copies a remote Object to the given path
-func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	err = f.client.FilesCopy((src.(*Object)).fs.mountID,
-		(src.(*Object)).fs.fullPath((src.(*Object)).remote),
-		f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantCopy
-	}
-	return f.NewObject(remote)
-}
-
-// Move moves a remote Object to the given path
-func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
-	srcObj := src.(*Object)
-	dstFullPath := f.fullPath(remote)
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	err = f.client.FilesMove(srcObj.fs.mountID,
-		srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath)
-	if err != nil {
-		return nil, fs.ErrorCantMove
-	}
-	return f.NewObject(remote)
-}
-
-// DirMove moves a remote directory to the given path
-func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
-	srcFs := src.(*Fs)
-	srcFullPath := srcFs.fullPath(srcRemote)
-	dstFullPath := f.fullPath(dstRemote)
-	if srcFs.mountID == f.mountID && srcFullPath == dstFullPath {
-		return fs.ErrorDirExists
-	}
-	dstDir := dir(dstFullPath)
-	err := f.mkdir(dstDir)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath)
-	if err != nil {
-		return fs.ErrorCantDirMove
-	}
-	return nil
-}
-
-// About reports space usage (with a MB precision)
-func (f *Fs) About() (*fs.Usage, error) {
-	mount, err := f.client.MountsDetails(f.mountID)
-	if err != nil {
-		return nil, err
-	}
-	return &fs.Usage{
-		Total:   fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024),
-		Used:    fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024),
-		Trashed: nil,
-		Other:   nil,
-		Free:    fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024),
-		Objects: nil,
-	}, nil
-}
-
-// Purge purges the complete Fs
-func (f *Fs) Purge() error {
-	err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath("")))
-	return err
-}
-
-// linkCreate is a Koofr API request for creating a public link
-type linkCreate struct {
-	Path string `json:"path"`
-}
-
-// link is a Koofr API response to creating a public link
-type link struct {
-	ID               string `json:"id"`
-	Name             string `json:"name"`
-	Path             string `json:"path"`
-	Counter          int64  `json:"counter"`
-	URL              string `json:"url"`
-	ShortURL         string `json:"shortUrl"`
-	Hash             string `json:"hash"`
-	Host             string `json:"host"`
-	HasPassword      bool   `json:"hasPassword"`
-	Password         string `json:"password"`
-	ValidFrom        int64  `json:"validFrom"`
-	ValidTo          int64  `json:"validTo"`
-	PasswordRequired bool   `json:"passwordRequired"`
-}
-
-// createLink makes a Koofr API call to create a public link
-func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) {
-	linkCreate := linkCreate{
-		Path: path,
-	}
-	linkData := link{}
-
-	request := httpclient.RequestData{
-		Method:         "POST",
-		Path:           "/api/v2/mounts/" + mountID + "/links",
-		ExpectedStatus: []int{http.StatusOK, http.StatusCreated},
-		ReqEncoding:    httpclient.EncodingJSON,
-		ReqValue:       linkCreate,
-		RespEncoding:   httpclient.EncodingJSON,
-		RespValue:      &linkData,
-	}
-
-	_, err := c.Request(&request)
-	if err != nil {
-		return nil, err
-	}
-	return &linkData, nil
-}
-
-// PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(remote string) (string, error) {
-	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
-	if err != nil {
-		return "", translateErrorsDir(err)
-	}
-	return linkData.ShortURL, nil
-}

backend/koofr/koofr_test.go

@@ -1,14 +0,0 @@
-package koofr_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestKoofr:",
-	})
-}

backend/local/about_unix.go

@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
-	bs := int64(s.Bsize) // nolint: unconvert
+	bs := int64(s.Bsize)
 	usage := &fs.Usage{
 		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
 		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use

backend/local/lchtimes.go

@@ -1,20 +0,0 @@
-// +build windows plan9
-
-package local
-
-import (
-	"time"
-)
-
-const haveLChtimes = false
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	// Does nothing
-	return nil
-}

backend/local/lchtimes_unix.go

@@ -1,28 +0,0 @@
-// +build !windows,!plan9
-
-package local
-
-import (
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const haveLChtimes = true
-
-// lChtimes changes the access and modification times of the named
-// link, similar to the Unix utime() or utimes() functions.
-//
-// The underlying filesystem may truncate or round the values to a
-// less precise time unit.
-// If there is an error, it will be of type *PathError.
-func lChtimes(name string, atime time.Time, mtime time.Time) error {
-	var utimes [2]unix.Timespec
-	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
-	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
-	if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
-		return &os.PathError{Op: "lchtimes", Path: name, Err: e}
-	}
-	return nil
-}

backend/local/local.go

@@ -2,7 +2,6 @@
 package local
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -17,19 +16,15 @@ import (
 	"unicode/utf8"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
 
 // Constants
 const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
-const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
 
 // Register with Fs
 func init() {
@@ -52,40 +47,19 @@ func init() {
 			ShortOpt: "L",
 			Advanced: true,
 		}, {
-			Name:     "links",
-			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
-			Default:  false,
-			NoPrefix: true,
-			ShortOpt: "l",
-			Advanced: true,
-		}, {
-			Name: "skip_links",
-			Help: `Don't warn about skipped symlinks.
-This flag disables warning messages on skipped symlinks or junction
-points, as you explicitly acknowledge that they should be skipped.`,
+			Name:     "skip_links",
+			Help:     "Don't warn about skipped symlinks.",
 			Default:  false,
 			NoPrefix: true,
 			Advanced: true,
 		}, {
-			Name: "no_unicode_normalization",
-			Help: `Don't apply unicode normalization to paths and filenames (Deprecated)
-
-This flag is deprecated now.  Rclone no longer normalizes unicode file
-names, but it compares them with unicode normalization in the sync
-routine instead.`,
+			Name:     "no_unicode_normalization",
+			Help:     "Don't apply unicode normalization to paths and filenames",
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name: "no_check_updated",
-			Help: `Don't check to see if the files change during upload
-
-Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
-
-However on some file systems this modification time check may fail (eg
-[Glusterfs #2206](https://github.com/ncw/rclone/issues/2206)) so this
-check can be disabled with this flag.`,
+			Name:     "no_check_updated",
+			Help:     "Don't check to see if the files change during upload",
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -102,13 +76,12 @@ check can be disabled with this flag.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks    bool `config:"copy_links"`
-	TranslateSymlinks bool `config:"links"`
-	SkipSymlinks      bool `config:"skip_links"`
-	NoUTFNorm         bool `config:"no_unicode_normalization"`
-	NoCheckUpdated    bool `config:"no_check_updated"`
-	NoUNC             bool `config:"nounc"`
-	OneFileSystem     bool `config:"one_file_system"`
+	FollowSymlinks bool `config:"copy_links"`
+	SkipSymlinks   bool `config:"skip_links"`
+	NoUTFNorm      bool `config:"no_unicode_normalization"`
+	NoCheckUpdated bool `config:"no_check_updated"`
+	NoUNC          bool `config:"nounc"`
+	OneFileSystem  bool `config:"one_file_system"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -130,20 +103,17 @@ type Fs struct {
 
 // Object represents a local filesystem object
 type Object struct {
-	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path - properly UTF-8 encoded - for rclone
-	path           string // The local path - may not be properly UTF-8 encoded - for OS
-	size           int64  // file metadata - always present
-	mode           os.FileMode
-	modTime        time.Time
-	hashes         map[hash.Type]string // Hashes
-	translatedLink bool                 // Is this object a translated link
+	fs      *Fs    // The Fs this object is part of
+	remote  string // The remote path - properly UTF-8 encoded - for rclone
+	path    string // The local path - may not be properly UTF-8 encoded - for OS
+	size    int64  // file metadata - always present
+	mode    os.FileMode
+	modTime time.Time
+	hashes  map[hash.Type]string // Hashes
 }
 
 // ------------------------------------------------------------
 
-var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
-
 // NewFs constructs an Fs from the path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -152,9 +122,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	if opt.TranslateSymlinks && opt.FollowSymlinks {
-		return nil, errLinksAndCopyLinks
-	}
 
 	if opt.NoUTFNorm {
 		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -182,7 +149,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	if err == nil && f.isRegular(fi.Mode()) {
+	if err == nil && fi.Mode().IsRegular() {
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -191,20 +158,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
-// Determine whether a file is a 'regular' file,
-// Symlinks are regular files, only if the TranslateSymlink
-// option is in-effect
-func (f *Fs) isRegular(mode os.FileMode) bool {
-	if !f.opt.TranslateSymlinks {
-		return mode.IsRegular()
-	}
-
-	// fi.Mode().IsRegular() tests that all mode bits are zero
-	// Since symlinks are accepted, test that all other bits are zero,
-	// except the symlink bit
-	return mode&os.ModeType&^os.ModeSymlink == 0
-}
-
 // Name of the remote (as passed into NewFs)
 func (f *Fs) Name() string {
 	return f.name
@@ -225,48 +178,28 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// caseInsensitive returns whether the remote is case insensitive or not
+// caseInsenstive returns whether the remote is case insensitive or not
 func (f *Fs) caseInsensitive() bool {
 	// FIXME not entirely accurate since you can have case
-	// sensitive Fses on darwin and case insensitive Fses on linux.
+	// sensitive Fses on darwin and case insenstive Fses on linux.
 	// Should probably check but that would involve creating a
 	// file in the remote to be most accurate which probably isn't
 	// desirable.
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
-// translateLink checks whether the remote is a translated link
-// and returns a new path, removing the suffix as needed,
-// It also returns whether this is a translated link at all
-//
-// for regular files, dstPath is returned unchanged
-func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
-	return newDstPath, isTranslatedLink
-}
-
 // newObject makes a half completed Object
 //
 // if dstPath is empty then it is made from remote
 func (f *Fs) newObject(remote, dstPath string) *Object {
-	translatedLink := false
-
 	if dstPath == "" {
 		dstPath = f.cleanPath(filepath.Join(f.root, remote))
 	}
 	remote = f.cleanRemote(remote)
-
-	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for dstPath
-		dstPath, translatedLink = translateLink(remote, dstPath)
-	}
-
 	return &Object{
-		fs:             f,
-		remote:         remote,
-		path:           dstPath,
-		translatedLink: translatedLink,
+		fs:     f,
+		remote: remote,
+		path:   dstPath,
 	}
 }
 
@@ -288,11 +221,6 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
-		// Handle the odd case, that a symlink was specified by name without the link suffix
-		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
-			return nil, fs.ErrorObjectNotFound
-		}
-
 	}
 	if o.mode.IsDir() {
 		return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -316,7 +244,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-
 	dir = f.dirNames.Load(dir)
 	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
 	remote := f.cleanRemote(dir)
@@ -353,13 +280,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				fi, err = os.Stat(newPath)
-				if os.IsNotExist(err) {
-					// Skip bad symlinks
-					err = fserrors.NoRetryError(errors.Wrap(err, "symlink"))
-					fs.Errorf(newRemote, "Listing error: %v", err)
-					accounting.Stats.Error(err)
-					continue
-				}
 				if err != nil {
 					return nil, err
 				}
@@ -373,10 +293,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 					entries = append(entries, d)
 				}
 			} else {
-				// Check whether this link should be translated
-				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
-					newRemote += linkSuffix
-				}
 				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
 				if err != nil {
 					return nil, err
@@ -590,7 +506,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.fs.isRegular(dstObj.mode) {
+	} else if !dstObj.mode.IsRegular() {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -712,13 +628,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		var in io.ReadCloser
-
-		if !o.translatedLink {
-			in, err = file.Open(o.path)
-		} else {
-			in, err = o.openTranslatedLink(0, -1)
-		}
+		in, err := os.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -749,12 +659,7 @@ func (o *Object) ModTime() time.Time {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	var err error
-	if o.translatedLink {
-		err = lChtimes(o.path, modTime, modTime)
-	} else {
-		err = os.Chtimes(o.path, modTime, modTime)
-	}
+	err := os.Chtimes(o.path, modTime, modTime)
 	if err != nil {
 		return err
 	}
@@ -772,7 +677,7 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
+	if mode&os.ModeSymlink != 0 {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}
@@ -833,16 +738,6 @@ func (file *localOpenFile) Close() (err error) {
 	return err
 }
 
-// Returns a ReadCloser() object that contains the contents of a symbolic link
-func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
-	// Read the link and return the destination  it as the contents of the object
-	linkdst, err := os.Readlink(o.path)
-	if err != nil {
-		return nil, err
-	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
-}
-
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
@@ -862,12 +757,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
-	// Handle a translated link
-	if o.translatedLink {
-		return o.openTranslatedLink(offset, limit)
-	}
-
-	fd, err := file.Open(o.path)
+	fd, err := os.Open(o.path)
 	if err != nil {
 		return
 	}
@@ -898,19 +788,8 @@ func (o *Object) mkdirAll() error {
 	return os.MkdirAll(dir, 0777)
 }
 
-type nopWriterCloser struct {
-	*bytes.Buffer
-}
-
-func (nwc nopWriterCloser) Close() error {
-	// noop
-	return nil
-}
-
 // Update the object from in with modTime and size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	var out io.WriteCloser
-
 	hashes := hash.Supported
 	for _, option := range options {
 		switch x := option.(type) {
@@ -924,23 +803,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	var symlinkData bytes.Buffer
-	// If the object is a regular file, create it.
-	// If it is a translated link, just read in the contents, and
-	// then create a symlink
-	if !o.translatedLink {
-		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-		if err != nil {
-			return err
-		}
-		// Pre-allocate the file for performance reasons
-		err = preAllocate(src.Size(), f)
-		if err != nil {
-			fs.Debugf(o, "Failed to pre-allocate: %v", err)
-		}
-		out = f
-	} else {
-		out = nopWriterCloser{&symlinkData}
+	out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return err
 	}
 
 	// Calculate the hash of the object we are reading as we go along
@@ -955,26 +820,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if err == nil {
 		err = closeErr
 	}
-
-	if o.translatedLink {
-		if err == nil {
-			// Remove any current symlink or file, if one exists
-			if _, err := os.Lstat(o.path); err == nil {
-				if removeErr := os.Remove(o.path); removeErr != nil {
-					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
-					return removeErr
-				}
-			}
-			// Use the contents for the copied object to create a symlink
-			err = os.Symlink(symlinkData.String(), o.path)
-		}
-
-		// only continue if symlink creation succeeded
-		if err != nil {
-			return err
-		}
-	}
-
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
 		if removeErr := os.Remove(o.path); removeErr != nil {

backend/local/local_internal_test.go

@@ -1,19 +1,13 @@
 package local
 
 import (
-	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
-	"runtime"
 	"testing"
 	"time"
 
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
-	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -44,13 +38,10 @@ func TestUpdatingCheck(t *testing.T) {
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
-	fd, err := file.Open(path.Join(r.LocalName, filePath))
+	fd, err := os.Open(path.Join(r.LocalName, filePath))
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}
-	defer func() {
-		require.NoError(t, fd.Close())
-	}()
 
 	fi, err := fd.Stat()
 	require.NoError(t, err)
@@ -81,108 +72,3 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 
 }
-
-func TestSymlink(t *testing.T) {
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	f := r.Flocal.(*Fs)
-	dir := f.root
-
-	// Write a file
-	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
-	file1 := r.WriteFile("file.txt", "hello", modTime1)
-
-	// Write a symlink
-	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
-	symlinkPath := filepath.Join(dir, "symlink.txt")
-	require.NoError(t, os.Symlink("file.txt", symlinkPath))
-	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
-
-	// Object viewed as symlink
-	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
-	if runtime.GOOS == "windows" {
-		file2.Size = 0 // symlinks are 0 length under Windows
-	}
-
-	// Object viewed as destination
-	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
-
-	// Check with no symlink flags
-	fstest.CheckItems(t, r.Flocal, file1)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-L" mode
-	f.opt.FollowSymlinks = true
-	f.opt.TranslateSymlinks = false
-	f.lstat = os.Stat
-
-	fstest.CheckItems(t, r.Flocal, file1, file2d)
-	fstest.CheckItems(t, r.Fremote)
-
-	// Set fs into "-l" mode
-	f.opt.FollowSymlinks = false
-	f.opt.TranslateSymlinks = true
-	f.lstat = os.Lstat
-
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2)
-	}
-
-	// Create a symlink
-	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
-	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
-	if runtime.GOOS == "windows" {
-		file3.Size = 0 // symlinks are 0 length under Windows
-	}
-	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
-	if haveLChtimes {
-		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
-	}
-
-	// Check it got the correct contents
-	symlinkPath = filepath.Join(dir, "symlink2.txt")
-	fi, err := os.Lstat(symlinkPath)
-	require.NoError(t, err)
-	assert.False(t, fi.Mode().IsRegular())
-	linkText, err := os.Readlink(symlinkPath)
-	require.NoError(t, err)
-	assert.Equal(t, "file.txt", linkText)
-
-	// Check that NewObject gets the correct object
-	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
-	require.NoError(t, err)
-	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
-	if runtime.GOOS != "windows" {
-		assert.Equal(t, int64(8), o.Size())
-	}
-
-	// Check that NewObject doesn't see the non suffixed version
-	_, err = r.Flocal.NewObject("symlink2.txt")
-	require.Equal(t, fs.ErrorObjectNotFound, err)
-
-	// Check reading the object
-	in, err := o.Open()
-	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt", string(contents))
-	require.NoError(t, in.Close())
-
-	// Check reading the object with range
-	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
-	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
-	require.NoError(t, err)
-	require.Equal(t, "file.txt"[2:5+1], string(contents))
-	require.NoError(t, in.Close())
-}
-
-func TestSymlinkError(t *testing.T) {
-	m := configmap.Simple{
-		"links":      "true",
-		"copy_links": "true",
-	}
-	_, err := NewFs("local", "/", m)
-	assert.Equal(t, errLinksAndCopyLinks, err)
-}

backend/local/preallocate_other.go

@@ -1,10 +0,0 @@
-//+build !windows,!linux
-
-package local
-
-import "os"
-
-// preAllocate the file for performance reasons
-func preAllocate(size int64, out *os.File) error {
-	return nil
-}

backend/local/preallocate_unix.go

@@ -1,22 +0,0 @@
-//+build linux
-
-package local
-
-import (
-	"os"
-
-	"golang.org/x/sys/unix"
-)
-
-// preAllocate the file for performance reasons
-func preAllocate(size int64, out *os.File) error {
-	if size <= 0 {
-		return nil
-	}
-	err := unix.Fallocate(int(out.Fd()), unix.FALLOC_FL_KEEP_SIZE, 0, size)
-	// FIXME could be doing something here
-	// if err == unix.ENOSPC {
-	// 	log.Printf("No space")
-	// }
-	return err
-}

backend/local/preallocate_windows.go

@@ -1,79 +0,0 @@
-//+build windows
-
-package local
-
-import (
-	"os"
-	"syscall"
-	"unsafe"
-
-	"github.com/pkg/errors"
-	"golang.org/x/sys/windows"
-)
-
-var (
-	ntdll                        = windows.NewLazySystemDLL("ntdll.dll")
-	ntQueryVolumeInformationFile = ntdll.NewProc("NtQueryVolumeInformationFile")
-	ntSetInformationFile         = ntdll.NewProc("NtSetInformationFile")
-)
-
-type fileAllocationInformation struct {
-	AllocationSize uint64
-}
-
-type fileFsSizeInformation struct {
-	TotalAllocationUnits     uint64
-	AvailableAllocationUnits uint64
-	SectorsPerAllocationUnit uint32
-	BytesPerSector           uint32
-}
-
-type ioStatusBlock struct {
-	Status, Information uintptr
-}
-
-// preAllocate the file for performance reasons
-func preAllocate(size int64, out *os.File) error {
-	if size <= 0 {
-		return nil
-	}
-
-	var (
-		iosb       ioStatusBlock
-		fsSizeInfo fileFsSizeInformation
-		allocInfo  fileAllocationInformation
-	)
-
-	// Query info about the block sizes on the file system
-	_, _, e1 := ntQueryVolumeInformationFile.Call(
-		uintptr(out.Fd()),
-		uintptr(unsafe.Pointer(&iosb)),
-		uintptr(unsafe.Pointer(&fsSizeInfo)),
-		uintptr(unsafe.Sizeof(fsSizeInfo)),
-		uintptr(3), // FileFsSizeInformation
-	)
-	if e1 != nil && e1 != syscall.Errno(0) {
-		return errors.Wrap(e1, "preAllocate NtQueryVolumeInformationFile failed")
-	}
-
-	// Calculate the allocation size
-	clusterSize := uint64(fsSizeInfo.BytesPerSector) * uint64(fsSizeInfo.SectorsPerAllocationUnit)
-	if clusterSize <= 0 {
-		return errors.Errorf("preAllocate clusterSize %d <= 0", clusterSize)
-	}
-	allocInfo.AllocationSize = (1 + uint64(size-1)/clusterSize) * clusterSize
-
-	// Ask for the allocation
-	_, _, e1 = ntSetInformationFile.Call(
-		uintptr(out.Fd()),
-		uintptr(unsafe.Pointer(&iosb)),
-		uintptr(unsafe.Pointer(&allocInfo)),
-		uintptr(unsafe.Sizeof(allocInfo)),
-		uintptr(19), // FileAllocationInformation
-	)
-	if e1 != nil && e1 != syscall.Errno(0) {
-		return errors.Wrap(e1, "preAllocate NtSetInformationFile failed")
-	}
-
-	return nil
-}

backend/local/read_device_unix.go

@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
 		fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
 		return devUnset
 	}
-	return uint64(statT.Dev) // nolint: unconvert
+	return uint64(statT.Dev)
 }

backend/mega/mega.go

@@ -63,20 +63,13 @@ func init() {
 			Required:   true,
 			IsPassword: true,
 		}, {
-			Name: "debug",
-			Help: `Output more debug from Mega.
-
-If this flag is set (along with -vv) it will print further debugging
-information from the mega backend.`,
+			Name:     "debug",
+			Help:     "Output more debug from Mega.",
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name: "hard_delete",
-			Help: `Delete files permanently rather than putting them into the trash.
-
-Normally the mega backend will put all deletions into the trash rather
-than permanently deleting them.  If you specify this then rclone will
-permanently delete objects instead.`,
+			Name:     "hard_delete",
+			Help:     "Delete files permanently rather than putting them into the trash.",
 			Default:  false,
 			Advanced: true,
 		}},
@@ -98,7 +91,7 @@ type Fs struct {
 	opt        Options      // parsed config options
 	features   *fs.Features // optional features
 	srv        *mega.Mega   // the connection to the server
-	pacer      *fs.Pacer    // pacer for API calls
+	pacer      *pacer.Pacer // pacer for API calls
 	rootNodeMu sync.Mutex   // mutex for _rootNode
 	_rootNode  *mega.Node   // root node - call findRoot to use this
 	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir
@@ -217,7 +210,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   srv,
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
@@ -402,27 +395,6 @@ func (f *Fs) clearRoot() {
 	//log.Printf("cleared root directory")
 }
 
-// CleanUp deletes all files currently in trash
-func (f *Fs) CleanUp() (err error) {
-	trash := f.srv.FS.GetTrash()
-	items := []*mega.Node{}
-	_, err = f.list(trash, func(item *mega.Node) bool {
-		items = append(items, item)
-		return false
-	})
-	if err != nil {
-		return errors.Wrap(err, "CleanUp failed to list items in trash")
-	}
-	// similar to f.deleteNode(trash) but with HardDelete as true
-	for _, item := range items {
-		err = f.pacer.Call(func() (bool, error) {
-			err = f.srv.Delete(item, true)
-			return shouldRetry(err)
-		})
-	}
-	return err
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -518,7 +490,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, object, leaf and error
+// Returns the dirNode, obect, leaf and error
 //
 // Used to create new objects
 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -544,10 +516,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(in, src, options...)
+		return exisitingObj, exisitingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -868,14 +840,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.deleteNode(srcDirNode)
 		if err != nil {
-			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1097,9 +1069,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
-	if size < 0 {
-		return errors.New("mega backend can't upload a file of unknown length")
-	}
 	//modTime := src.ModTime()
 	remote := o.Remote()
 
@@ -1150,7 +1119,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return errors.Wrap(err, "failed to finish upload")
 	}
 
-	// If the upload succeeded and the original object existed, then delete it
+	// If the upload succeded and the original object existed, then delete it
 	if o.info != nil {
 		err = o.fs.deleteNode(o.info)
 		if err != nil {

backend/onedrive/api/types.go

@@ -9,9 +9,6 @@ import (
 
 const (
 	timeFormat = `"` + time.RFC3339 + `"`
-
-	// PackageTypeOneNote is the package type value for OneNote files
-	PackageTypeOneNote = "oneNote"
 )
 
 // Error is returned from one drive when things go wrong
@@ -25,7 +22,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := e.ErrorInfo.Code
 	if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +32,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Identity represents an identity of an actor. For example, and actor
@@ -94,10 +91,9 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 // ItemReference groups data needed to reference a OneDrive item
 // across the service into a single structure.
 type ItemReference struct {
-	DriveID   string `json:"driveId"`   // Unique identifier for the Drive that contains the item.	Read-only.
-	ID        string `json:"id"`        // Unique identifier for the item.	Read/Write.
-	Path      string `json:"path"`      // Path that used to navigate to the item.	Read/Write.
-	DriveType string `json:"driveType"` // Type of the drive,	Read-Only
+	DriveID string `json:"driveId"` // Unique identifier for the Drive that contains the item.	Read-only.
+	ID      string `json:"id"`      // Unique identifier for the item.	Read/Write.
+	Path    string `json:"path"`    // Path that used to navigate to the item.	Read/Write.
 }
 
 // RemoteItemFacet groups data needed to reference a OneDrive remote item
@@ -110,7 +106,6 @@ type RemoteItemFacet struct {
 	LastModifiedDateTime Timestamp            `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only.
 	Folder               *FolderFacet         `json:"folder"`               // Folder metadata, if the item is a folder. Read-only.
 	File                 *FileFacet           `json:"file"`                 // File metadata, if the item is a file. Read-only.
-	Package              *PackageFacet        `json:"package"`              // If present, indicates that this item is a package instead of a folder or file. Packages are treated like files in some contexts and folders in others. Read-only.
 	FileSystemInfo       *FileSystemInfoFacet `json:"fileSystemInfo"`       // File system information on client. Read-write.
 	ParentReference      *ItemReference       `json:"parentReference"`      // Parent information, if the item has a parent. Read-write.
 	Size                 int64                `json:"size"`                 // Size of the item in bytes. Read-only.
@@ -151,13 +146,6 @@ type FileSystemInfoFacet struct {
 type DeletedFacet struct {
 }
 
-// PackageFacet indicates that a DriveItem is the top level item
-// in a "package" or a collection of items that should be treated as a collection instead of individual items.
-// `oneNote` is the only currently defined value.
-type PackageFacet struct {
-	Type string `json:"type"`
-}
-
 // Item represents metadata for an item in OneDrive
 type Item struct {
 	ID                   string               `json:"id"`                   // The unique identifier of the item within the Drive. Read-only.
@@ -181,7 +169,6 @@ type Item struct {
 	//	Audio                *AudioFacet          `json:"audio"`                // Audio metadata, if the item is an audio file. Read-only.
 	//	Video                *VideoFacet          `json:"video"`                // Video metadata, if the item is a video. Read-only.
 	//	Location             *LocationFacet       `json:"location"`             // Location metadata, if the item has location data. Read-only.
-	Package *PackageFacet `json:"package"` // If present, indicates that this item is a package instead of a folder or file. Packages are treated like files in some contexts and folders in others. Read-only.
 	Deleted *DeletedFacet `json:"deleted"` // Information about the deleted state of the item. Read-only.
 }
 
@@ -250,28 +237,6 @@ type MoveItemRequest struct {
 	FileSystemInfo  *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"`  // File system information on client. Read-write.
 }
 
-//CreateShareLinkRequest is the request to create a sharing link
-//Always Type:view and Scope:anonymous for public sharing
-type CreateShareLinkRequest struct {
-	Type  string `json:"type"`            //Link type in View, Edit or Embed
-	Scope string `json:"scope,omitempty"` //Optional. Scope in anonymousi, organization
-}
-
-//CreateShareLinkResponse is the response from CreateShareLinkRequest
-type CreateShareLinkResponse struct {
-	ID    string   `json:"id"`
-	Roles []string `json:"roles"`
-	Link  struct {
-		Type        string `json:"type"`
-		Scope       string `json:"scope"`
-		WebURL      string `json:"webUrl"`
-		Application struct {
-			ID          string `json:"id"`
-			DisplayName string `json:"displayName"`
-		} `json:"application"`
-	} `json:"link"`
-}
-
 // AsyncOperationStatus provides information on the status of a asynchronous job progress.
 //
 // The following API calls return AsyncOperationStatus resources:
@@ -279,13 +244,13 @@ type CreateShareLinkResponse struct {
 // Copy Item
 // Upload From URL
 type AsyncOperationStatus struct {
+	Operation          string  `json:"operation"`          // The type of job being run.
 	PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
 	Status             string  `json:"status"`             // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
 }
 
 // GetID returns a normalized ID of the item
 // If DriveID is known it will be prefixed to the ID with # seperator
-// Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {
 		return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID
@@ -295,9 +260,9 @@ func (i *Item) GetID() string {
 	return i.ID
 }
 
-// GetDriveID returns a normalized ParentReference of the item
+// GetDriveID returns a normalized ParentReferance of the item
 func (i *Item) GetDriveID() string {
-	return i.GetParentReference().DriveID
+	return i.GetParentReferance().DriveID
 }
 
 // GetName returns a normalized Name of the item
@@ -316,24 +281,6 @@ func (i *Item) GetFolder() *FolderFacet {
 	return i.Folder
 }
 
-// GetPackage returns a normalized Package of the item
-func (i *Item) GetPackage() *PackageFacet {
-	if i.IsRemote() && i.RemoteItem.Package != nil {
-		return i.RemoteItem.Package
-	}
-	return i.Package
-}
-
-// GetPackageType returns the package type of the item if available,
-// otherwise ""
-func (i *Item) GetPackageType() string {
-	pack := i.GetPackage()
-	if pack == nil {
-		return ""
-	}
-	return pack.Type
-}
-
 // GetFile returns a normalized File of the item
 func (i *Item) GetFile() *FileFacet {
 	if i.IsRemote() && i.RemoteItem.File != nil {
@@ -398,8 +345,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
 	return i.LastModifiedDateTime
 }
 
-// GetParentReference returns a normalized ParentReference of the item
-func (i *Item) GetParentReference() *ItemReference {
+// GetParentReferance returns a normalized ParentReferance of the item
+func (i *Item) GetParentReferance() *ItemReference {
 	if i.IsRemote() && i.ParentReference == nil {
 		return i.RemoteItem.ParentReference
 	}

backend/onedrive/onedrive_test.go

@@ -1,10 +1,10 @@
 // Test OneDrive filesystem interface
-package onedrive
+package onedrive_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/onedrive"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -12,15 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestOneDrive:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			CeilChunkSize: fstests.NextMultipleOf(chunkSizeMultiple),
-		},
+		NilObject:  (*onedrive.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
-	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -21,7 +20,6 @@ import (
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/readers"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 )
@@ -65,7 +63,7 @@ type Fs struct {
 	opt      Options            // parsed options
 	features *fs.Features       // optional features
 	srv      *rest.Client       // the connection to the server
-	pacer    *fs.Pacer          // To pace and retry the API calls
+	pacer    *pacer.Pacer       // To pace and retry the API calls
 	session  UserSessionInfo    // contains the session data
 	dirCache *dircache.DirCache // Map of directory path to directory id
 }
@@ -119,7 +117,7 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -144,7 +142,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 
 	f.dirCache = dircache.New(root, "0", f)
@@ -179,17 +177,17 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		// Assume it is a file
 		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, "0", &tempF)
-		tempF.root = newRoot
+		newF := *f
+		newF.dirCache = dircache.New(newRoot, "0", &newF)
+		newF.root = newRoot
 
 		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(false)
+		err = newF.dirCache.FindRoot(false)
 		if err != nil {
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(remote, nil)
+		_, err := newF.newObjectWithInfo(remote, nil)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -197,13 +195,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			}
 			return nil, err
 		}
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/ncw/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
 		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
+		return &newF, fs.ErrorIsFile
 	}
 	return f, nil
 }
@@ -287,6 +280,9 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -782,7 +778,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, folder.FolderID)
-		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
+		d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
 		entries = append(entries, d)
 	}
@@ -929,9 +925,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// resp.Body.Close()
 	// fs.Debugf(nil, "PostOpen: %#v", openResponse)
 
-	// 10 MB chunks size
+	// 1 MB chunks size
 	chunkSize := int64(1024 * 1024 * 10)
-	buf := make([]byte, int(chunkSize))
 	chunkOffset := int64(0)
 	remainingBytes := size
 	chunkCounter := 0
@@ -944,19 +939,14 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		remainingBytes -= currentChunkSize
 		fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
 
-		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, currentChunkSize)
 		err = o.fs.pacer.Call(func() (bool, error) {
-			// seek to the start in case this is a retry
-			if _, err = chunk.Seek(0, io.SeekStart); err != nil {
-				return false, err
-			}
 			var formBody bytes.Buffer
 			w := multipart.NewWriter(&formBody)
 			fw, err := w.CreateFormFile("file_data", o.remote)
 			if err != nil {
 				return false, err
 			}
-			if _, err = io.Copy(fw, chunk); err != nil {
+			if _, err = io.CopyN(fw, in, currentChunkSize); err != nil {
 				return false, err
 			}
 			// Add session_id
@@ -1087,7 +1077,7 @@ func (o *Object) readMetaData() (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts := rest.Opts{
 			Method: "GET",
-			Path:   "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)),
+			Path:   "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + rest.URLPathEscape(replaceReservedChars(leaf)),
 		}
 		resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList)
 		return o.fs.shouldRetry(resp, err)

backend/opendrive/types.go

@@ -13,7 +13,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }

backend/pcloud/api/types.go

@@ -41,7 +41,7 @@ type Error struct {
 	ErrorString string `json:"error"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
 }
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
 	return e
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // Item describes a folder or a file as returned by Get Folder Items and others
@@ -161,6 +161,7 @@ type UserInfo struct {
 	PublicLinkQuota       int64  `json:"publiclinkquota"`
 	Email                 string `json:"email"`
 	UserID                int    `json:"userid"`
+	Result                int    `json:"result"`
 	Quota                 int64  `json:"quota"`
 	TrashRevretentionDays int    `json:"trashrevretentiondays"`
 	Premium               bool   `json:"premium"`

backend/pcloud/pcloud.go

@@ -95,7 +95,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
+	pacer        *pacer.Pacer       // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
@@ -246,7 +246,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	root = parsePath(root)
 	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to configure Pcloud")
+		log.Fatalf("Failed to configure Pcloud: %v", err)
 	}
 
 	f := &Fs{
@@ -254,7 +254,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         false,
@@ -276,16 +276,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		// Assume it is a file
 		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
-		tempF.root = newRoot
+		newF := *f
+		newF.dirCache = dircache.New(newRoot, rootID, &newF)
+		newF.root = newRoot
 		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(false)
+		err = newF.dirCache.FindRoot(false)
 		if err != nil {
 			// No root so return old f
 			return f, nil
 		}
-		_, err := tempF.newObjectWithInfo(remote, nil)
+		_, err := newF.newObjectWithInfo(remote, nil)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				// File doesn't exist so return old f
@@ -293,13 +293,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			}
 			return nil, err
 		}
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/ncw/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
 		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
+		return &newF, fs.ErrorIsFile
 	}
 	return f, nil
 }
@@ -385,7 +380,7 @@ func fileIDtoNumber(fileID string) string {
 	if len(fileID) > 0 && fileID[0] == 'f' {
 		return fileID[1:]
 	}
-	fs.Debugf(nil, "Invalid file id %q", fileID)
+	fs.Debugf(nil, "Invalid filee id %q", fileID)
 	return fileID
 }
 

backend/qingstor/qingstor.go

@@ -69,57 +69,17 @@ func init() {
 			}},
 		}, {
 			Name:     "connection_retries",
-			Help:     "Number of connection retries.",
+			Help:     "Number of connnection retries.",
 			Default:  3,
 			Advanced: true,
-		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-The minimum is 0 and the maximum is 5GB.`,
-			Default:  defaultUploadCutoff,
-			Advanced: true,
-		}, {
-			Name: "chunk_size",
-			Help: `Chunk size to use for uploading.
-
-When uploading files larger than upload_cutoff they will be uploaded
-as multipart uploads using this chunk size.
-
-Note that "--qingstor-upload-concurrency" chunks of this size are buffered
-in memory per transfer.
-
-If you are transferring large files over high speed links and you have
-enough memory, then increasing this will speed up the transfers.`,
-			Default:  minChunkSize,
-			Advanced: true,
-		}, {
-			Name: "upload_concurrency",
-			Help: `Concurrency for multipart uploads.
-
-This is the number of chunks of the same file that are uploaded
-concurrently.
-
-NB if you set this to > 1 then the checksums of multpart uploads
-become corrupted (the uploads themselves are not corrupted though).
-
-If you are uploading small numbers of large file over high speed link
-and these uploads do not fully utilize your bandwidth, then increasing
-this may help to speed up the transfers.`,
-			Default:  1,
-			Advanced: true,
 		}},
 	})
 }
 
 // Constants
 const (
-	listLimitSize       = 1000                   // Number of items to read at once
-	maxSizeForCopy      = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
-	minChunkSize        = fs.SizeSuffix(minMultiPartSize)
-	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
-	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
+	listLimitSize  = 1000                   // Number of items to read at once
+	maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
 )
 
 // Globals
@@ -132,15 +92,12 @@ func timestampToTime(tp int64) time.Time {
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth           bool          `config:"env_auth"`
-	AccessKeyID       string        `config:"access_key_id"`
-	SecretAccessKey   string        `config:"secret_access_key"`
-	Endpoint          string        `config:"endpoint"`
-	Zone              string        `config:"zone"`
-	ConnectionRetries int           `config:"connection_retries"`
-	UploadCutoff      fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize         fs.SizeSuffix `config:"chunk_size"`
-	UploadConcurrency int           `config:"upload_concurrency"`
+	EnvAuth           bool   `config:"env_auth"`
+	AccessKeyID       string `config:"access_key_id"`
+	SecretAccessKey   string `config:"secret_access_key"`
+	Endpoint          string `config:"endpoint"`
+	Zone              string `config:"zone"`
+	ConnectionRetries int    `config:"connection_retries"`
 }
 
 // Fs represents a remote qingstor server
@@ -270,36 +227,6 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
 	return qs.Init(cf)
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxUploadCutoff {
-		return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -308,14 +235,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "qingstor: chunk size")
-	}
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, errors.Wrap(err, "qingstor: upload cutoff")
-	}
 	bucket, key, err := qsParsePath(root)
 	if err != nil {
 		return nil, err
@@ -449,7 +368,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	_, err = bucketInit.PutObject(key, &req)
 	if err != nil {
-		fs.Debugf(f, "Copy Failed, API Error: %v", err)
+		fs.Debugf(f, "Copied Faild, API Error: %v", err)
 		return nil, err
 	}
 	return f.NewObject(remote)
@@ -756,7 +675,7 @@ func (f *Fs) Mkdir(dir string) error {
 		}
 		switch *statistics.Status {
 		case "deleted":
-			fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
+			fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
 			time.Sleep(time.Second * 1)
 			retries++
 			continue
@@ -875,7 +794,7 @@ func (o *Object) readMetaData() (err error) {
 	fs.Debugf(o, "Read metadata of key: %s", key)
 	resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
 	if err != nil {
-		fs.Debugf(o, "Read metadata failed, API Error: %v", err)
+		fs.Debugf(o, "Read metadata faild, API Error: %v", err)
 		if e, ok := err.(*qsErr.QingStorError); ok {
 			if e.StatusCode == http.StatusNotFound {
 				return fs.ErrorObjectNotFound
@@ -994,24 +913,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	mimeType := fs.MimeType(src)
 
 	req := uploadInput{
-		body:        in,
-		qsSvc:       o.fs.svc,
-		bucket:      o.fs.bucket,
-		zone:        o.fs.zone,
-		key:         key,
-		mimeType:    mimeType,
-		partSize:    int64(o.fs.opt.ChunkSize),
-		concurrency: o.fs.opt.UploadConcurrency,
+		body:     in,
+		qsSvc:    o.fs.svc,
+		bucket:   o.fs.bucket,
+		zone:     o.fs.zone,
+		key:      key,
+		mimeType: mimeType,
 	}
 	uploader := newUploader(&req)
 
-	size := src.Size()
-	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
-	if multipart {
-		err = uploader.upload()
-	} else {
-		err = uploader.singlePartUpload(in, size)
-	}
+	err = uploader.upload()
 	if err != nil {
 		return err
 	}

backend/qingstor/qingstor_test.go

@@ -2,12 +2,12 @@
 
 // +build !plan9
 
-package qingstor
+package qingstor_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/qingstor"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -15,19 +15,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestQingStor:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: minChunkSize,
-		},
+		NilObject:  (*qingstor.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/qingstor/upload.go

@@ -143,7 +143,7 @@ func (u *uploader) init() {
 
 		// Try to adjust partSize if it is too small and account for
 		// integer division truncation.
-		if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
+		if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
 			// Add one to the part size to account for remainders
 			// during the size calculation. e.g odd number of bytes.
 			u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -152,18 +152,18 @@ func (u *uploader) init() {
 }
 
 // singlePartUpload upload a single object that contentLength less than "defaultUploadPartSize"
-func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
+func (u *uploader) singlePartUpload(buf io.ReadSeeker) error {
 	bucketInit, _ := u.bucketInit()
 
 	req := qs.PutObjectInput{
-		ContentLength: &size,
+		ContentLength: &u.readerPos,
 		ContentType:   &u.cfg.mimeType,
 		Body:          buf,
 	}
 
 	_, err := bucketInit.PutObject(u.cfg.key, &req)
 	if err == nil {
-		fs.Debugf(u, "Upload single object finished")
+		fs.Debugf(u, "Upload single objcet finished")
 	}
 	return err
 }
@@ -179,13 +179,13 @@ func (u *uploader) upload() error {
 	// Do one read to determine if we have more than one part
 	reader, _, err := u.nextReader()
 	if err == io.EOF { // single part
-		fs.Debugf(u, "Uploading as single part object to QingStor")
-		return u.singlePartUpload(reader, u.readerPos)
+		fs.Debugf(u, "Tried to upload a singile object to QingStor")
+		return u.singlePartUpload(reader)
 	} else if err != nil {
 		return errors.Errorf("read upload data failed: %s", err)
 	}
 
-	fs.Debugf(u, "Uploading as multi-part object to QingStor")
+	fs.Debugf(u, "Treied to upload a multi-part object to QingStor")
 	mu := multiUploader{uploader: u}
 	return mu.multiPartUpload(reader)
 }
@@ -261,7 +261,7 @@ func (mu *multiUploader) initiate() error {
 	req := qs.InitiateMultipartUploadInput{
 		ContentType: &mu.cfg.mimeType,
 	}
-	fs.Debugf(mu, "Initiating a multi-part upload")
+	fs.Debugf(mu, "Tried to initiate a multi-part upload")
 	rsp, err := bucketInit.InitiateMultipartUpload(mu.cfg.key, &req)
 	if err == nil {
 		mu.uploadID = rsp.UploadID
@@ -279,12 +279,12 @@ func (mu *multiUploader) send(c chunk) error {
 		ContentLength: &c.size,
 		Body:          c.buffer,
 	}
-	fs.Debugf(mu, "Uploading a part to QingStor with partNumber %d and partSize %d", c.partNumber, c.size)
+	fs.Debugf(mu, "Tried to upload a part to QingStor that partNumber %d and partSize %d", c.partNumber, c.size)
 	_, err := bucketInit.UploadMultipart(mu.cfg.key, &req)
 	if err != nil {
 		return err
 	}
-	fs.Debugf(mu, "Done uploading part partNumber %d and partSize %d", c.partNumber, c.size)
+	fs.Debugf(mu, "Upload part finished that partNumber %d and partSize %d", c.partNumber, c.size)
 
 	mu.mtx.Lock()
 	defer mu.mtx.Unlock()
@@ -304,7 +304,7 @@ func (mu *multiUploader) list() error {
 	req := qs.ListMultipartInput{
 		UploadID: mu.uploadID,
 	}
-	fs.Debugf(mu, "Reading multi-part details")
+	fs.Debugf(mu, "Tried to list a multi-part")
 	rsp, err := bucketInit.ListMultipart(mu.cfg.key, &req)
 	if err == nil {
 		mu.objectParts = rsp.ObjectParts
@@ -331,7 +331,7 @@ func (mu *multiUploader) complete() error {
 		ObjectParts: mu.objectParts,
 		ETag:        &md5String,
 	}
-	fs.Debugf(mu, "Completing multi-part object")
+	fs.Debugf(mu, "Tried to complete a multi-part")
 	_, err = bucketInit.CompleteMultipartUpload(mu.cfg.key, &req)
 	if err == nil {
 		fs.Debugf(mu, "Complete multi-part finished")
@@ -348,7 +348,7 @@ func (mu *multiUploader) abort() error {
 		req := qs.AbortMultipartUploadInput{
 			UploadID: uploadID,
 		}
-		fs.Debugf(mu, "Aborting multi-part object %q", *uploadID)
+		fs.Debugf(mu, "Tried to abort a multi-part")
 		_, err = bucketInit.AbortMultipartUpload(mu.cfg.key, &req)
 	}
 
@@ -392,14 +392,6 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
 		var nextChunkLen int
 		reader, nextChunkLen, err = mu.nextReader()
 		if err != nil && err != io.EOF {
-			// empty ch
-			go func() {
-				for range ch {
-				}
-			}()
-			// Wait for all goroutines finish
-			close(ch)
-			mu.wg.Wait()
 			return err
 		}
 		if nextChunkLen == 0 && partNumber > 0 {

backend/s3/s3.go

@@ -39,11 +39,9 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
-	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/ncw/swift"
 	"github.com/pkg/errors"
@@ -53,7 +51,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
+		Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
@@ -61,9 +59,6 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "AWS",
 				Help:  "Amazon Web Services (AWS) S3",
-			}, {
-				Value: "Alibaba",
-				Help:  "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
 			}, {
 				Value: "Ceph",
 				Help:  "Ceph Object Storage",
@@ -79,9 +74,6 @@ func init() {
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
-			}, {
-				Value: "Netease",
-				Help:  "Netease Object Storage (NOS)",
 			}, {
 				Value: "Wasabi",
 				Help:  "Wasabi Object Storage",
@@ -131,9 +123,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -159,7 +148,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba",
+			Provider: "!AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -237,10 +226,10 @@ func init() {
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.softlayer.net",
-				Help:  "Great Britain Endpoint",
+				Help:  "Great Britan Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
-				Help:  "Great Britain Private Endpoint",
+				Help:  "Great Britan Private Endpoint",
 			}, {
 				Value: "s3.ap-geo.objectstorage.softlayer.net",
 				Help:  "APAC Cross Regional Endpoint",
@@ -278,75 +267,12 @@ func init() {
 				Value: "s3.tor01.objectstorage.service.networklayer.com",
 				Help:  "Toronto Single Site Private Endpoint",
 			}},
-		}, {
-			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
-			Name:     "endpoint",
-			Help:     "Endpoint for OSS API.",
-			Provider: "Alibaba",
-			Examples: []fs.OptionExample{{
-				Value: "oss-cn-hangzhou.aliyuncs.com",
-				Help:  "East China 1 (Hangzhou)",
-			}, {
-				Value: "oss-cn-shanghai.aliyuncs.com",
-				Help:  "East China 2 (Shanghai)",
-			}, {
-				Value: "oss-cn-qingdao.aliyuncs.com",
-				Help:  "North China 1 (Qingdao)",
-			}, {
-				Value: "oss-cn-beijing.aliyuncs.com",
-				Help:  "North China 2 (Beijing)",
-			}, {
-				Value: "oss-cn-zhangjiakou.aliyuncs.com",
-				Help:  "North China 3 (Zhangjiakou)",
-			}, {
-				Value: "oss-cn-huhehaote.aliyuncs.com",
-				Help:  "North China 5 (Huhehaote)",
-			}, {
-				Value: "oss-cn-shenzhen.aliyuncs.com",
-				Help:  "South China 1 (Shenzhen)",
-			}, {
-				Value: "oss-cn-hongkong.aliyuncs.com",
-				Help:  "Hong Kong (Hong Kong)",
-			}, {
-				Value: "oss-us-west-1.aliyuncs.com",
-				Help:  "US West 1 (Silicon Valley)",
-			}, {
-				Value: "oss-us-east-1.aliyuncs.com",
-				Help:  "US East 1 (Virginia)",
-			}, {
-				Value: "oss-ap-southeast-1.aliyuncs.com",
-				Help:  "Southeast Asia Southeast 1 (Singapore)",
-			}, {
-				Value: "oss-ap-southeast-2.aliyuncs.com",
-				Help:  "Asia Pacific Southeast 2 (Sydney)",
-			}, {
-				Value: "oss-ap-southeast-3.aliyuncs.com",
-				Help:  "Southeast Asia Southeast 3 (Kuala Lumpur)",
-			}, {
-				Value: "oss-ap-southeast-5.aliyuncs.com",
-				Help:  "Asia Pacific Southeast 5 (Jakarta)",
-			}, {
-				Value: "oss-ap-northeast-1.aliyuncs.com",
-				Help:  "Asia Pacific Northeast 1 (Japan)",
-			}, {
-				Value: "oss-ap-south-1.aliyuncs.com",
-				Help:  "Asia Pacific South 1 (Mumbai)",
-			}, {
-				Value: "oss-eu-central-1.aliyuncs.com",
-				Help:  "Central Europe 1 (Frankfurt)",
-			}, {
-				Value: "oss-eu-west-1.aliyuncs.com",
-				Help:  "West Europe (London)",
-			}, {
-				Value: "oss-me-east-1.aliyuncs.com",
-				Help:  "Middle East 1 (Dubai)",
-			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS",
 			Examples: []fs.OptionExample{{
-				Value:    "objects-us-east-1.dream.io",
+				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
 			}, {
@@ -363,11 +289,7 @@ func init() {
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "s3.wasabisys.com",
-				Help:     "Wasabi US East endpoint",
-				Provider: "Wasabi",
-			}, {
-				Value:    "s3.us-west-1.wasabisys.com",
-				Help:     "Wasabi US West endpoint",
+				Help:     "Wasabi Object Storage",
 				Provider: "Wasabi",
 			}},
 		}, {
@@ -395,9 +317,6 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
-			}, {
-				Value: "eu-north-1",
-				Help:  "EU (Stockholm) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -450,7 +369,7 @@ func init() {
 				Help:  "US East Region Flex",
 			}, {
 				Value: "us-south-standard",
-				Help:  "US South Region Standard",
+				Help:  "US Sout hRegion Standard",
 			}, {
 				Value: "us-south-vault",
 				Help:  "US South Region Vault",
@@ -474,16 +393,16 @@ func init() {
 				Help:  "EU Cross Region Flex",
 			}, {
 				Value: "eu-gb-standard",
-				Help:  "Great Britain Standard",
+				Help:  "Great Britan Standard",
 			}, {
 				Value: "eu-gb-vault",
-				Help:  "Great Britain Vault",
+				Help:  "Great Britan Vault",
 			}, {
 				Value: "eu-gb-cold",
-				Help:  "Great Britain Cold",
+				Help:  "Great Britan Cold",
 			}, {
 				Value: "eu-gb-flex",
-				Help:  "Great Britain Flex",
+				Help:  "Great Britan Flex",
 			}, {
 				Value: "ap-standard",
 				Help:  "APAC Standard",
@@ -524,17 +443,10 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS",
 		}, {
 			Name: "acl",
-			Help: `Canned ACL used when creating buckets and storing or copying objects.
-
-This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
-
-For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
-
-Note that this ACL is applied when server side copying objects as S3
-doesn't copy the ACL from the source but rather writes a fresh one.`,
+			Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl",
 			Examples: []fs.OptionExample{{
 				Value:    "private",
 				Help:     "Owner gets FULL_CONTROL. No one else has access rights (default).",
@@ -576,28 +488,6 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 				Help:     "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS",
 				Provider: "IBMCOS",
 			}},
-		}, {
-			Name: "bucket_acl",
-			Help: `Canned ACL used when creating buckets.
-
-For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
-
-Note that this ACL is applied when only when creating buckets.  If it
-isn't set then "acl" is used instead.`,
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: "private",
-				Help:  "Owner gets FULL_CONTROL. No one else has access rights (default).",
-			}, {
-				Value: "public-read",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
-			}, {
-				Value: "public-read-write",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
-			}, {
-				Value: "authenticated-read",
-				Help:  "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
-			}},
 		}, {
 			Name:     "server_side_encryption",
 			Help:     "The server-side encryption algorithm used when storing this object in S3.",
@@ -625,7 +515,7 @@ isn't set then "acl" is used instead.`,
 			}},
 		}, {
 			Name:     "storage_class",
-			Help:     "The storage class to use when storing new objects in S3.",
+			Help:     "The storage class to use when storing objects in S3.",
 			Provider: "AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -642,52 +532,11 @@ isn't set then "acl" is used instead.`,
 			}, {
 				Value: "ONEZONE_IA",
 				Help:  "One Zone Infrequent Access storage class",
-			}, {
-				Value: "GLACIER",
-				Help:  "Glacier storage class",
-			}, {
-				Value: "DEEP_ARCHIVE",
-				Help:  "Glacier Deep Archive storage class",
 			}},
 		}, {
-			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
-			Name:     "storage_class",
-			Help:     "The storage class to use when storing new objects in OSS.",
-			Provider: "Alibaba",
-			Examples: []fs.OptionExample{{
-				Value: "",
-				Help:  "Default",
-			}, {
-				Value: "STANDARD",
-				Help:  "Standard storage class",
-			}, {
-				Value: "GLACIER",
-				Help:  "Archive storage mode.",
-			}, {
-				Value: "STANDARD_IA",
-				Help:  "Infrequent access storage mode.",
-			}},
-		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-The minimum is 0 and the maximum is 5GB.`,
-			Default:  defaultUploadCutoff,
-			Advanced: true,
-		}, {
-			Name: "chunk_size",
-			Help: `Chunk size to use for uploading.
-
-When uploading files larger than upload_cutoff they will be uploaded
-as multipart uploads using this chunk size.
-
-Note that "--s3-upload-concurrency" chunks of this size are buffered
-in memory per transfer.
-
-If you are transferring large files over high speed links and you have
-enough memory, then increasing this will speed up the transfers.`,
-			Default:  minChunkSize,
+			Name:     "chunk_size",
+			Help:     "Chunk size to use for uploading",
+			Default:  fs.SizeSuffix(s3manager.MinUploadPartSize),
 			Advanced: true,
 		}, {
 			Name:     "disable_checksum",
@@ -697,89 +546,50 @@ enough memory, then increasing this will speed up the transfers.`,
 		}, {
 			Name:     "session_token",
 			Help:     "An AWS session token",
+			Hide:     fs.OptionHideBoth,
 			Advanced: true,
 		}, {
-			Name: "upload_concurrency",
-			Help: `Concurrency for multipart uploads.
-
-This is the number of chunks of the same file that are uploaded
-concurrently.
-
-If you are uploading small numbers of large file over high speed link
-and these uploads do not fully utilize your bandwidth, then increasing
-this may help to speed up the transfers.`,
-			Default:  4,
+			Name:     "upload_concurrency",
+			Help:     "Concurrency for multipart uploads.",
+			Default:  2,
 			Advanced: true,
 		}, {
-			Name: "force_path_style",
-			Help: `If true use path style access if false use virtual hosted style.
-
-If this is true (the default) then rclone will use path style access,
-if false then rclone will use virtual path style. See [the AWS S3
-docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
-for more info.
-
-Some providers (eg Aliyun OSS or Netease COS) require this set to false.`,
+			Name:     "force_path_style",
+			Help:     "If true use path style access if false use virtual hosted style.\nSome providers (eg Aliyun OSS or Netease COS) require this.",
 			Default:  true,
 			Advanced: true,
-		}, {
-			Name: "v2_auth",
-			Help: `If true use v2 authentication.
-
-If this is false (the default) then rclone will use v4 authentication.
-If it is set then rclone will use v2 authentication.
-
-Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     "use_accelerate_endpoint",
-			Provider: "AWS",
-			Help: `If true use the AWS S3 accelerated endpoint.
-
-See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
-			Default:  false,
-			Advanced: true,
 		}},
 	})
 }
 
 // Constants
 const (
-	metaMtime           = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	metaMD5Hash         = "Md5chksum"                   // the meta key to store md5hash in
-	listChunkSize       = 1000                          // number of items to read at once
-	maxRetries          = 10                            // number of retries to make of operations
-	maxSizeForCopy      = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
-	maxFileSize         = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
-	minChunkSize        = fs.SizeSuffix(s3manager.MinUploadPartSize)
-	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
-	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
-	minSleep            = 10 * time.Millisecond // In case of error, start at 10ms sleep.
+	metaMtime      = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
+	metaMD5Hash    = "Md5chksum"                   // the meta key to store md5hash in
+	listChunkSize  = 1000                          // number of items to read at once
+	maxRetries     = 10                            // number of retries to make of operations
+	maxSizeForCopy = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
+	maxFileSize    = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
 )
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider              string        `config:"provider"`
-	EnvAuth               bool          `config:"env_auth"`
-	AccessKeyID           string        `config:"access_key_id"`
-	SecretAccessKey       string        `config:"secret_access_key"`
-	Region                string        `config:"region"`
-	Endpoint              string        `config:"endpoint"`
-	LocationConstraint    string        `config:"location_constraint"`
-	ACL                   string        `config:"acl"`
-	BucketACL             string        `config:"bucket_acl"`
-	ServerSideEncryption  string        `config:"server_side_encryption"`
-	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
-	StorageClass          string        `config:"storage_class"`
-	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum       bool          `config:"disable_checksum"`
-	SessionToken          string        `config:"session_token"`
-	UploadConcurrency     int           `config:"upload_concurrency"`
-	ForcePathStyle        bool          `config:"force_path_style"`
-	V2Auth                bool          `config:"v2_auth"`
-	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
+	Provider             string        `config:"provider"`
+	EnvAuth              bool          `config:"env_auth"`
+	AccessKeyID          string        `config:"access_key_id"`
+	SecretAccessKey      string        `config:"secret_access_key"`
+	Region               string        `config:"region"`
+	Endpoint             string        `config:"endpoint"`
+	LocationConstraint   string        `config:"location_constraint"`
+	ACL                  string        `config:"acl"`
+	ServerSideEncryption string        `config:"server_side_encryption"`
+	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
+	StorageClass         string        `config:"storage_class"`
+	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
+	DisableChecksum      bool          `config:"disable_checksum"`
+	SessionToken         string        `config:"session_token"`
+	UploadConcurrency    int           `config:"upload_concurrency"`
+	ForcePathStyle       bool          `config:"force_path_style"`
 }
 
 // Fs represents a remote s3 server
@@ -794,8 +604,6 @@ type Fs struct {
 	bucketOKMu    sync.Mutex       // mutex to protect bucket OK
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
-	pacer         *fs.Pacer        // To pace the API calls
-	srv           *http.Client     // a plain http client
 }
 
 // Object describes a s3 object
@@ -841,45 +649,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// retryErrorCodes is a slice of error codes that we will retry
-// See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-var retryErrorCodes = []int{
-	// 409, // Conflict - various states that could be resolved on a retry
-	503, // Service Unavailable/Slow Down - "Reduce your request rate"
-}
-
-//S3 is pretty resilient, and the built in retry handling is probably sufficient
-// as it should notice closed connections and timeouts which are the most likely
-// sort of failure modes
-func (f *Fs) shouldRetry(err error) (bool, error) {
-	// If this is an awserr object, try and extract more useful information to determine if we should retry
-	if awsError, ok := err.(awserr.Error); ok {
-		// Simple case, check the original embedded error in case it's generically retryable
-		if fserrors.ShouldRetry(awsError.OrigErr()) {
-			return true, err
-		}
-		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
-		if reqErr, ok := err.(awserr.RequestFailure); ok {
-			// 301 if wrong region for bucket
-			if reqErr.StatusCode() == http.StatusMovedPermanently {
-				urfbErr := f.updateRegionForBucket()
-				if urfbErr != nil {
-					fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
-					return false, err
-				}
-				return true, err
-			}
-			for _, e := range retryErrorCodes {
-				if reqErr.StatusCode() == e {
-					return true, err
-				}
-			}
-		}
-	}
-	// Ok, not an awserr, check for generic failure conditions
-	return fserrors.ShouldRetry(err), err
-}
-
 // Pattern to match a s3 path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -953,39 +722,17 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
-		opt.ForcePathStyle = false
-	}
 	awsConfig := aws.NewConfig().
+		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
+		WithEndpoint(opt.Endpoint).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
-		WithS3ForcePathStyle(opt.ForcePathStyle).
-		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
-	if opt.Region != "" {
-		awsConfig.WithRegion(opt.Region)
-	}
-	if opt.Endpoint != "" {
-		awsConfig.WithEndpoint(opt.Endpoint)
-	}
-
+		WithS3ForcePathStyle(opt.ForcePathStyle)
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
-	awsSessionOpts := session.Options{
-		Config: *awsConfig,
-	}
-	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
-		// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
-		awsSessionOpts.SharedConfigState = session.SharedConfigEnable
-		// The session constructor (aws/session/mergeConfigSrcs) will only use the user's preferred credential source
-		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
-		awsSessionOpts.Config.Credentials = nil
-	}
-	ses, err := session.NewSessionWithOptions(awsSessionOpts)
-	if err != nil {
-		return nil, nil, err
-	}
-	c := s3.New(ses)
-	if opt.V2Auth || opt.Region == "other-v2-signature" {
+	ses := session.New()
+	c := s3.New(ses, awsConfig)
+	if opt.Region == "other-v2-signature" {
 		fs.Debugf(nil, "Using v2 auth")
 		signer := func(req *request.Request) {
 			// Ignore AnonymousCredentials object
@@ -1001,36 +748,6 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	return c, ses, nil
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxUploadCutoff {
-		return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -1039,24 +756,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "s3: chunk size")
-	}
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, errors.Wrap(err, "s3: upload cutoff")
+	if opt.ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) {
+		return nil, errors.Errorf("s3 chunk size (%v) must be >= %v", opt.ChunkSize, fs.SizeSuffix(s3manager.MinUploadPartSize))
 	}
 	bucket, directory, err := s3ParsePath(root)
 	if err != nil {
 		return nil, err
 	}
-	if opt.ACL == "" {
-		opt.ACL = "private"
-	}
-	if opt.BucketACL == "" {
-		opt.BucketACL = opt.ACL
-	}
 	c, ses, err := s3Connection(opt)
 	if err != nil {
 		return nil, err
@@ -1068,8 +774,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		c:      c,
 		bucket: bucket,
 		ses:    ses,
-		pacer:  fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
-		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -1083,10 +787,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			Bucket: &f.bucket,
 			Key:    &directory,
 		}
-		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.c.HeadObject(&req)
-			return f.shouldRetry(err)
-		})
+		_, err = f.c.HeadObject(&req)
 		if err == nil {
 			f.root = path.Dir(directory)
 			if f.root == "." {
@@ -1135,51 +836,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// Gets the bucket location
-func (f *Fs) getBucketLocation() (string, error) {
-	req := s3.GetBucketLocationInput{
-		Bucket: &f.bucket,
-	}
-	var resp *s3.GetBucketLocationOutput
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(err)
-	})
-	if err != nil {
-		return "", err
-	}
-	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
-}
-
-// Updates the region for the bucket by reading the region from the
-// bucket then updating the session.
-func (f *Fs) updateRegionForBucket() error {
-	region, err := f.getBucketLocation()
-	if err != nil {
-		return errors.Wrap(err, "reading bucket location failed")
-	}
-	if aws.StringValue(f.c.Config.Endpoint) != "" {
-		return errors.Errorf("can't set region to %q as endpoint is set", region)
-	}
-	if aws.StringValue(f.c.Config.Region) == region {
-		return errors.Errorf("region is already %q - not updating", region)
-	}
-
-	// Make a new session with the new region
-	oldRegion := f.opt.Region
-	f.opt.Region = region
-	c, ses, err := s3Connection(&f.opt)
-	if err != nil {
-		return errors.Wrap(err, "creating new session failed")
-	}
-	f.c = c
-	f.ses = ses
-
-	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
-	return nil
-}
-
 // listFn is called from list to handle an object.
 type listFn func(remote string, object *s3.Object, isDirectory bool) error
 
@@ -1208,12 +864,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 			MaxKeys:   &maxKeys,
 			Marker:    marker,
 		}
-		var resp *s3.ListObjectsOutput
-		var err error
-		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.c.ListObjects(&req)
-			return f.shouldRetry(err)
-		})
+		resp, err := f.c.ListObjects(&req)
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
 				if awsErr.StatusCode() == http.StatusNotFound {
@@ -1338,11 +989,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 		return nil, fs.ErrorListBucketRequired
 	}
 	req := s3.ListBucketsInput{}
-	var resp *s3.ListBucketsOutput
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.ListBuckets(&req)
-		return f.shouldRetry(err)
-	})
+	resp, err := f.c.ListBuckets(&req)
 	if err != nil {
 		return nil, err
 	}
@@ -1427,10 +1074,7 @@ func (f *Fs) dirExists() (bool, error) {
 	req := s3.HeadBucketInput{
 		Bucket: &f.bucket,
 	}
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.c.HeadBucket(&req)
-		return f.shouldRetry(err)
-	})
+	_, err := f.c.HeadBucket(&req)
 	if err == nil {
 		return true, nil
 	}
@@ -1460,17 +1104,14 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	req := s3.CreateBucketInput{
 		Bucket: &f.bucket,
-		ACL:    &f.opt.BucketACL,
+		ACL:    &f.opt.ACL,
 	}
 	if f.opt.LocationConstraint != "" {
 		req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
 			LocationConstraint: &f.opt.LocationConstraint,
 		}
 	}
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.c.CreateBucket(&req)
-		return f.shouldRetry(err)
-	})
+	_, err := f.c.CreateBucket(&req)
 	if err, ok := err.(awserr.Error); ok {
 		if err.Code() == "BucketAlreadyOwnedByYou" {
 			err = nil
@@ -1479,7 +1120,6 @@ func (f *Fs) Mkdir(dir string) error {
 	if err == nil {
 		f.bucketOK = true
 		f.bucketDeleted = false
-		fs.Infof(f, "Bucket created with ACL %q", *req.ACL)
 	}
 	return err
 }
@@ -1496,14 +1136,10 @@ func (f *Fs) Rmdir(dir string) error {
 	req := s3.DeleteBucketInput{
 		Bucket: &f.bucket,
 	}
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.c.DeleteBucket(&req)
-		return f.shouldRetry(err)
-	})
+	_, err := f.c.DeleteBucket(&req)
 	if err == nil {
 		f.bucketOK = false
 		f.bucketDeleted = true
-		fs.Infof(f, "Bucket deleted")
 	}
 	return err
 }
@@ -1543,24 +1179,11 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	source := pathEscape(srcFs.bucket + "/" + srcFs.root + srcObj.remote)
 	req := s3.CopyObjectInput{
 		Bucket:            &f.bucket,
-		ACL:               &f.opt.ACL,
 		Key:               &key,
 		CopySource:        &source,
 		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
 	}
-	if f.opt.ServerSideEncryption != "" {
-		req.ServerSideEncryption = &f.opt.ServerSideEncryption
-	}
-	if f.opt.SSEKMSKeyID != "" {
-		req.SSEKMSKeyId = &f.opt.SSEKMSKeyID
-	}
-	if f.opt.StorageClass != "" {
-		req.StorageClass = &f.opt.StorageClass
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.c.CopyObject(&req)
-		return f.shouldRetry(err)
-	})
+	_, err = f.c.CopyObject(&req)
 	if err != nil {
 		return nil, err
 	}
@@ -1637,12 +1260,7 @@ func (o *Object) readMetaData() (err error) {
 		Bucket: &o.fs.bucket,
 		Key:    &key,
 	}
-	var resp *s3.HeadObjectOutput
-	err = o.fs.pacer.Call(func() (bool, error) {
-		var err error
-		resp, err = o.fs.c.HeadObject(&req)
-		return o.fs.shouldRetry(err)
-	})
+	resp, err := o.fs.c.HeadObject(&req)
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
 			if awsErr.StatusCode() == http.StatusNotFound {
@@ -1726,19 +1344,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
 		Metadata:          o.meta,
 		MetadataDirective: &directive,
 	}
-	if o.fs.opt.ServerSideEncryption != "" {
-		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-	}
-	if o.fs.opt.SSEKMSKeyID != "" {
-		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-	}
-	if o.fs.opt.StorageClass != "" {
-		req.StorageClass = &o.fs.opt.StorageClass
-	}
-	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err := o.fs.c.CopyObject(&req)
-		return o.fs.shouldRetry(err)
-	})
+	_, err = o.fs.c.CopyObject(&req)
 	return err
 }
 
@@ -1765,12 +1371,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 			}
 		}
 	}
-	var resp *s3.GetObjectOutput
-	err = o.fs.pacer.Call(func() (bool, error) {
-		var err error
-		resp, err = o.fs.c.GetObject(&req)
-		return o.fs.shouldRetry(err)
-	})
+	resp, err := o.fs.c.GetObject(&req)
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
 			return nil, errors.Errorf("Object in GLACIER, restore first: %v", key)
@@ -1791,46 +1392,38 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	modTime := src.ModTime()
 	size := src.Size()
 
-	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
-	var uploader *s3manager.Uploader
-	if multipart {
-		uploader = s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-			u.Concurrency = o.fs.opt.UploadConcurrency
-			u.LeavePartsOnError = false
-			u.S3 = o.fs.c
-			u.PartSize = int64(o.fs.opt.ChunkSize)
+	uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
+		u.Concurrency = o.fs.opt.UploadConcurrency
+		u.LeavePartsOnError = false
+		u.S3 = o.fs.c
+		u.PartSize = int64(o.fs.opt.ChunkSize)
 
-			if size == -1 {
-				// Make parts as small as possible while still being able to upload to the
-				// S3 file size limit. Rounded up to nearest MB.
-				u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
-				return
-			}
-			// Adjust PartSize until the number of parts is small enough.
-			if size/u.PartSize >= s3manager.MaxUploadParts {
-				// Calculate partition size rounded up to the nearest MB
-				u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
-			}
-		})
-	}
+		if size == -1 {
+			// Make parts as small as possible while still being able to upload to the
+			// S3 file size limit. Rounded up to nearest MB.
+			u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
+			return
+		}
+		// Adjust PartSize until the number of parts is small enough.
+		if size/u.PartSize >= s3manager.MaxUploadParts {
+			// Calculate partition size rounded up to the nearest MB
+			u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
+		}
+	})
 
 	// Set the mtime in the meta data
 	metadata := map[string]*string{
 		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
 	}
 
-	// read the md5sum if available for non multpart and if
-	// disable checksum isn't present.
-	var md5sum string
-	if !multipart || !o.fs.opt.DisableChecksum {
+	if !o.fs.opt.DisableChecksum && size > uploader.PartSize {
 		hash, err := src.Hash(hash.MD5)
+
 		if err == nil && matchMd5.MatchString(hash) {
 			hashBytes, err := hex.DecodeString(hash)
+
 			if err == nil {
-				md5sum = base64.StdEncoding.EncodeToString(hashBytes)
-				if multipart {
-					metadata[metaMD5Hash] = &md5sum
-				}
+				metadata[metaMD5Hash] = aws.String(base64.StdEncoding.EncodeToString(hashBytes))
 			}
 		}
 	}
@@ -1839,98 +1432,27 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	mimeType := fs.MimeType(src)
 
 	key := o.fs.root + o.remote
-	if multipart {
-		req := s3manager.UploadInput{
-			Bucket:      &o.fs.bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &key,
-			Body:        in,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-			//ContentLength: &size,
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = uploader.Upload(&req)
-			return o.fs.shouldRetry(err)
-		})
-		if err != nil {
-			return err
-		}
-	} else {
-		req := s3.PutObjectInput{
-			Bucket:      &o.fs.bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &key,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-		}
-		if md5sum != "" {
-			req.ContentMD5 = &md5sum
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
-
-		// Create the request
-		putObj, _ := o.fs.c.PutObjectRequest(&req)
-
-		// Sign it so we can upload using a presigned request.
-		//
-		// Note the SDK doesn't currently support streaming to
-		// PutObject so we'll use this work-around.
-		url, headers, err := putObj.PresignRequest(15 * time.Minute)
-		if err != nil {
-			return errors.Wrap(err, "s3 upload: sign request")
-		}
-
-		// Set request to nil if empty so as not to make chunked encoding
-		if size == 0 {
-			in = nil
-		}
-
-		// create the vanilla http request
-		httpReq, err := http.NewRequest("PUT", url, in)
-		if err != nil {
-			return errors.Wrap(err, "s3 upload: new request")
-		}
-
-		// set the headers we signed and the length
-		httpReq.Header = headers
-		httpReq.ContentLength = size
-
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			resp, err := o.fs.srv.Do(httpReq)
-			if err != nil {
-				return o.fs.shouldRetry(err)
-			}
-			body, err := rest.ReadBody(resp)
-			if err != nil {
-				return o.fs.shouldRetry(err)
-			}
-			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
-				return false, nil
-			}
-			err = errors.Errorf("s3 upload: %s: %s", resp.Status, body)
-			return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-		})
-		if err != nil {
-			return err
-		}
+	req := s3manager.UploadInput{
+		Bucket:      &o.fs.bucket,
+		ACL:         &o.fs.opt.ACL,
+		Key:         &key,
+		Body:        in,
+		ContentType: &mimeType,
+		Metadata:    metadata,
+		//ContentLength: &size,
+	}
+	if o.fs.opt.ServerSideEncryption != "" {
+		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
+	}
+	if o.fs.opt.SSEKMSKeyID != "" {
+		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
+	}
+	if o.fs.opt.StorageClass != "" {
+		req.StorageClass = &o.fs.opt.StorageClass
+	}
+	_, err = uploader.Upload(&req)
+	if err != nil {
+		return err
 	}
 
 	// Read the metadata from the newly created object
@@ -1946,10 +1468,7 @@ func (o *Object) Remove() error {
 		Bucket: &o.fs.bucket,
 		Key:    &key,
 	}
-	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := o.fs.c.DeleteObject(&req)
-		return o.fs.shouldRetry(err)
-	})
+	_, err := o.fs.c.DeleteObject(&req)
 	return err
 }
 

backend/s3/s3_test.go

@@ -1,10 +1,10 @@
 // Test S3 filesystem interface
-package s3
+package s3_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/s3"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -12,19 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestS3:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: minChunkSize,
-		},
+		NilObject:  (*s3.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/s3/v2sign.go

@@ -44,7 +44,16 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 	req.Header.Set("Date", date)
 
 	// Sort out URI
-	uri := req.URL.EscapedPath()
+	uri := req.URL.Opaque
+	if uri != "" {
+		if strings.HasPrefix(uri, "//") {
+			// Strip off //host/uri
+			uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
+			req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused
+		}
+	} else {
+		uri = req.URL.Path
+	}
 	if uri == "" {
 		uri = "/"
 	}

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 
@@ -14,7 +14,6 @@ import (
 	"os/user"
 	"path"
 	"regexp"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -29,7 +28,7 @@ import (
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
-	sshagent "github.com/xanzy/ssh-agent"
+	"github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/time/rate"
 )
@@ -67,22 +66,7 @@ func init() {
 			IsPassword: true,
 		}, {
 			Name: "key_file",
-			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
-		}, {
-			Name: "key_file_pass",
-			Help: `The passphrase to decrypt the PEM-encoded private key file.
-
-Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
-in the new OpenSSH format can't be used.`,
-			IsPassword: true,
-		}, {
-			Name: "key_use_agent",
-			Help: `When set forces the usage of the ssh-agent.
-
-When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
-requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
-when the ssh-agent contains many keys.`,
-			Default: false,
+			Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
 		}, {
 			Name:    "use_insecure_cipher",
 			Help:    "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
@@ -106,20 +90,9 @@ when the ssh-agent contains many keys.`,
 			Help:     "Allow asking for SFTP password when needed.",
 			Advanced: true,
 		}, {
-			Name:    "path_override",
-			Default: "",
-			Help: `Override path used by SSH connection.
-
-This allows checksum calculation when SFTP and SSH paths are
-different. This issue affects among others Synology NAS boxes.
-
-Shared folders can be found in directories representing volumes
-
-    rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
-
-Home directory can be found in a shared folder called "home"
-
-    rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory`,
+			Name:     "path_override",
+			Default:  "",
+			Help:     "Override path used by SSH connection.",
 			Advanced: true,
 		}, {
 			Name:     "set_modtime",
@@ -138,8 +111,6 @@ type Options struct {
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
 	KeyFile           string `config:"key_file"`
-	KeyFilePass       string `config:"key_file_pass"`
-	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
 	AskPassword       bool   `config:"ask_password"`
@@ -316,18 +287,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
-// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
-// variables afterwards.
-func shellExpand(s string) string {
-	if s != "" {
-		if s[0] == '~' {
-			s = "${HOME}" + s[1:]
-		}
-		s = os.ExpandEnv(s)
-	}
-	return s
-}
-
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -355,9 +314,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
-	keyFile := shellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
-	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
+	if opt.Pass == "" && opt.KeyFile == "" {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -366,46 +324,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't read ssh agent signers")
 		}
-		if keyFile != "" {
-			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to read public key file")
-			}
-			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to parse public key file")
-			}
-			pubM := pub.Marshal()
-			found := false
-			for _, s := range signers {
-				if bytes.Equal(pubM, s.PublicKey().Marshal()) {
-					sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
-					found = true
-					break
-				}
-			}
-			if !found {
-				return nil, errors.New("private key not found in the ssh-agent")
-			}
-		} else {
-			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
-		}
+		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
 	}
 
 	// Load key file if specified
-	if keyFile != "" {
-		key, err := ioutil.ReadFile(keyFile)
+	if opt.KeyFile != "" {
+		key, err := ioutil.ReadFile(opt.KeyFile)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read private key file")
 		}
-		clearpass := ""
-		if opt.KeyFilePass != "" {
-			clearpass, err = obscure.Reveal(opt.KeyFilePass)
-			if err != nil {
-				return nil, err
-			}
-		}
-		signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
+		signer, err := ssh.ParsePrivateKey(key)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}
@@ -428,12 +356,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
-	return NewFsWithConnection(name, root, opt, sshConfig)
-}
-
-// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
-// the host specified in the ssh.ClientConfig
-func NewFsWithConnection(name string, root string, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
 		name:      name,
 		root:      root,
@@ -572,13 +494,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
 		if !info.Mode().IsRegular() {
-			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
-				if !os.IsNotExist(err) {
-					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
-				}
-				info = oldInfo
+				return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
 			}
 		}
 		if info.IsDir() {
@@ -665,22 +583,12 @@ func (f *Fs) Mkdir(dir string) error {
 
 // Rmdir removes the root directory of the Fs object
 func (f *Fs) Rmdir(dir string) error {
-	// Check to see if directory is empty as some servers will
-	// delete recursively with RemoveDirectory
-	entries, err := f.List(dir)
-	if err != nil {
-		return errors.Wrap(err, "Rmdir")
-	}
-	if len(entries) != 0 {
-		return fs.ErrorDirectoryNotEmpty
-	}
-	// Remove the directory
 	root := path.Join(f.root, dir)
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return errors.Wrap(err, "Rmdir")
 	}
-	err = c.sftpClient.RemoveDirectory(root)
+	err = c.sftpClient.Remove(root)
 	f.putSftpConnection(&c, err)
 	return err
 }
@@ -814,47 +722,6 @@ func (f *Fs) Hashes() hash.Set {
 	return set
 }
 
-// About gets usage stats
-func (f *Fs) About() (*fs.Usage, error) {
-	c, err := f.getSftpConnection()
-	if err != nil {
-		return nil, errors.Wrap(err, "About get SFTP connection")
-	}
-	session, err := c.sshClient.NewSession()
-	f.putSftpConnection(&c, err)
-	if err != nil {
-		return nil, errors.Wrap(err, "About put SFTP connection")
-	}
-
-	var stdout, stderr bytes.Buffer
-	session.Stdout = &stdout
-	session.Stderr = &stderr
-	escapedPath := shellEscape(f.root)
-	if f.opt.PathOverride != "" {
-		escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
-	}
-	if len(escapedPath) == 0 {
-		escapedPath = "/"
-	}
-	err = session.Run("df -k " + escapedPath)
-	if err != nil {
-		_ = session.Close()
-		return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.")
-	}
-	_ = session.Close()
-
-	usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes())
-	if usageTotal < 0 || usageUsed < 0 || usageAvail < 0 {
-		return nil, errors.Wrap(err, "About failed to parse information")
-	}
-	usage := &fs.Usage{
-		Total: fs.NewUsageValue(usageTotal),
-		Used:  fs.NewUsageValue(usageUsed),
-		Free:  fs.NewUsageValue(usageAvail),
-	}
-	return usage, nil
-}
-
 // Fs is the filesystem this remote sftp file object is located within
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -891,10 +758,6 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		return "", hash.ErrUnsupported
 	}
 
-	if o.fs.opt.DisableHashCheck {
-		return "", nil
-	}
-
 	c, err := o.fs.getSftpConnection()
 	if err != nil {
 		return "", errors.Wrap(err, "Hash get SFTP connection")
@@ -945,34 +808,6 @@ func parseHash(bytes []byte) string {
 	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
 }
 
-// Parses the byte array output from the SSH session
-// returned by an invocation of df into
-// the disk size, used space, and avaliable space on the disk, in that order.
-// Only works when `df` has output info on only one disk
-func parseUsage(bytes []byte) (int64, int64, int64) {
-	lines := strings.Split(string(bytes), "\n")
-	if len(lines) < 2 {
-		return -1, -1, -1
-	}
-	split := strings.Fields(lines[1])
-	if len(split) < 6 {
-		return -1, -1, -1
-	}
-	spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
-	if err != nil {
-		return -1, -1, -1
-	}
-	spaceUsed, err := strconv.ParseInt(split[2], 10, 64)
-	if err != nil {
-		return -1, -1, -1
-	}
-	spaceAvail, err := strconv.ParseInt(split[3], 10, 64)
-	if err != nil {
-		return -1, -1, -1
-	}
-	return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
-}
-
 // Size returns the size in bytes of the remote sftp file
 func (o *Object) Size() int64 {
 	return o.size

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 
@@ -35,17 +35,3 @@ func TestParseHash(t *testing.T) {
 		assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
 	}
 }
-
-func TestParseUsage(t *testing.T) {
-	for i, test := range []struct {
-		sshOutput string
-		usage     [3]int64
-	}{
-		{"Filesystem     1K-blocks     Used Available Use% Mounted on\n/dev/root       91283092 81111888  10154820  89% /", [3]int64{93473886208, 83058573312, 10398535680}},
-		{"Filesystem     1K-blocks  Used Available Use% Mounted on\ntmpfs             818256  1636    816620   1% /run", [3]int64{837894144, 1675264, 836218880}},
-		{"Filesystem   1024-blocks     Used Available Capacity iused      ifree %iused  Mounted on\n/dev/disk0s2   244277768 94454848 149566920    39%  997820 4293969459    0%   /", [3]int64{250140434432, 96721764352, 153156526080}},
-	} {
-		gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput))
-		assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
-	}
-}

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9
+// +build plan9 !go1.9
 
 package sftp