Version v1.43.1

- Reduced size of the social menu and increased the size of the content
- Added scrollable property to the index menus
- Fixed code wrapping issue

Fixes #2103

Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>

.appveyor.yml

@@ -0,0 +1,46 @@
+version: "{build}"
+
+os: Windows Server 2012 R2
+
+clone_folder: c:\gopath\src\github.com\ncw\rclone
+
+environment:
+  GOPATH: C:\gopath
+  CPATH: C:\Program Files (x86)\WinFsp\inc\fuse
+  ORIGPATH: '%PATH%'
+  NOCCPATH: C:\MinGW\bin;%GOPATH%\bin;%PATH%
+  PATHCC64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%NOCCPATH%
+  PATHCC32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%NOCCPATH%
+  PATH: '%PATHCC64%'
+  RCLONE_CONFIG_PASS:
+    secure: HbzxSy9zQ8NYWN9NNPf6ALQO9Q0mwRNqwehsLcOEHy0=
+
+install:
+- choco install winfsp -y
+- choco install zip -y
+- copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe
+
+build_script:
+- echo %PATH%
+- echo %GOPATH%
+- go version
+- go env
+- go install
+- go build
+- make log_since_last_release > %TEMP%\git-log.txt
+- make version > %TEMP%\version
+- set /p RCLONE_VERSION=<%TEMP%\version
+- set PATH=%PATHCC32%
+- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/386" -cgo -tags cmount %RCLONE_VERSION%
+- set PATH=%PATHCC64%
+- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/amd64" -cgo -no-clean -tags cmount %RCLONE_VERSION%
+
+test_script:
+- make GOTAGS=cmount quicktest
+
+artifacts:
+- path: rclone.exe
+- path: build/*-v*.zip
+
+deploy_script:
+- IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload

.circleci/config.yml

@@ -0,0 +1,34 @@
+version: 2
+
+jobs:
+
+  build:
+    machine: true
+
+    working_directory: ~/.go_workspace/src/github.com/ncw/rclone
+
+    steps:
+    - checkout
+
+    - run:
+        name: Cross-compile rclone
+        command: |
+            docker pull billziss/xgo-cgofuse
+            go get -v github.com/karalabe/xgo
+            xgo \
+                --image=billziss/xgo-cgofuse \
+                --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+                -tags cmount \
+                .
+            xgo \
+                --targets=android/*,ios/* \
+                .
+
+    - run:
+        name: Prepare artifacts
+        command: |
+            mkdir -p /tmp/rclone.dist
+            cp -R rclone-* /tmp/rclone.dist
+
+    - store_artifacts:
+        path: /tmp/rclone.dist

.gitattributes

@@ -1,7 +0,0 @@
-# Ignore generated files in GitHub language statistics and diffs
-/MANUAL.* linguist-generated=true
-/rclone.1 linguist-generated=true
-
-# Don't fiddle with the line endings of test data
-**/testdata/** -text
-**/test/** -text

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-github: [ncw]
-patreon: njcw
-liberapay: ncw
-custom: ["https://rclone.org/donate/"]

.github/ISSUE_TEMPLATE.md

@@ -1,31 +0,0 @@
-<!--
-
-Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
-
-If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
-
-    https://forum.rclone.org/
-
-instead of filing an issue for a quick response.
-
-If you are reporting a bug or asking for a new feature then please use one of the templates here:
-
-    https://github.com/rclone/rclone/issues/new
-
-otherwise fill in the form below.
-
-Thank you
-
-The Rclone Developers
-
--->
-
-
-#### Output of `rclone version`
-
-
-
-#### Describe the issue
-
-
-

.github/ISSUE_TEMPLATE/Bug.md

@@ -1,67 +0,0 @@
----
-name: Bug report
-about: Report a problem with rclone
----
-
-<!--
-
-We understand you are having a problem with rclone; we want to help you with that!
-
-**STOP and READ**
-**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
-Please show the effort you've put in to solving the problem and please be specific.
-People are volunteering their time to help! Low effort posts are not likely to get good answers!
-
-If you think you might have found a bug, try to replicate it with the latest beta (or stable).
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
-
-If you can still replicate it or just got a question then please use the rclone forum:
-
-    https://forum.rclone.org/
-
-for a quick response instead of filing an issue on this repo.
-
-If nothing else helps, then please fill in the info below which helps us help you.
-
-**DO NOT REDACT** any information except passwords/keys/personal info.
-
-You should use 3 backticks to begin and end your paste to make it readable.
-
-Make sure to include a log obtained with '-vv'.
-
-You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
-
-Thank you
-
-The Rclone Developers
-
--->
-
-
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
-
-#### What is the problem you are having with rclone?
-
-
-
-#### What is your rclone version (output from `rclone version`)
-
-
-
-#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
-
-
-
-#### Which cloud storage system are you using? (e.g. Google Drive)
-
-
-
-#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
-
-
-
-#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
-
-

.github/ISSUE_TEMPLATE/Feature.md

@@ -1,44 +0,0 @@
----
-name: Feature request
-about: Suggest a new feature or enhancement for rclone
----
-
-<!--
-
-Welcome :-)
-
-So you've got an idea to improve rclone? We love that!
-You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
-
-Probably the latest beta (or stable) release has your feature, so try to update your rclone.
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
-
-If it still isn't there, here is a checklist of things to do:
-
-  1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum: https://forum.rclone.org/
-  3. Make a feature request issue (this is the right place!).
-  4. Be prepared to get involved making the feature :-)
-
-Looking forward to your great idea!
-
-The Rclone Developers
-
--->
-
-
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
-
-#### What is your current rclone version (output from `rclone version`)?
-
-
-
-#### What problem are you are trying to solve?
-
-
-
-#### How do you think rclone should be changed to solve that?
-
-

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Rclone Forum Community Support
-    url: https://forum.rclone.org/
-    about: Please ask and answer questions here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,29 +0,0 @@
-<!--
-Thank you very much for contributing code or documentation to rclone! Please
-fill out the following questions to make it easier for us to review your
-changes.
-
-You do not need to check all the boxes below all at once, feel free to take
-your time and add more commits. If you're done and ready for review, please
-check the last box.
--->
-
-#### What is the purpose of this change?
-
-<!--
-Describe the changes here
--->
-
-#### Was the change discussed in an issue or in the forum before?
-
-<!--
-Link issues and relevant forum posts here.
--->
-
-#### Checklist
-
-- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
-- [ ] I have added tests for all changes in this PR if appropriate.
-- [ ] I have added documentation for the changes if appropriate.
-- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
-- [ ] I'm done, this Pull Request is ready for review :-)

.github/workflows/build.yml

@@ -1,312 +0,0 @@
----
-# Github Actions build for rclone
-# -*- compile-command: "yamllint -f parsable build.yml" -*-
-
-name: build
-
-# Trigger the workflow on push or pull request
-on:
-  push:
-    branches:
-      - '*'
-    tags:
-      - '*'
-  pull_request:
-
-jobs:
-  build:
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.13', 'go1.14', 'go1.15']
-
-        include:
-          - job_name: linux
-            os: ubuntu-latest
-            go: '1.16.x'
-            gotags: cmount
-            build_flags: '-include "^linux/"'
-            check: true
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: mac_amd64
-            os: macOS-latest
-            go: '1.16.x'
-            gotags: 'cmount'
-            build_flags: '-include "^darwin/amd64" -cgo'
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: mac_arm64
-            os: macOS-latest
-            go: '1.16.x'
-            gotags: 'cmount'
-            build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -macos-sdk macosx11.1 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
-            deploy: true
-
-          - job_name: windows_amd64
-            os: windows-latest
-            go: '1.16.x'
-            gotags: cmount
-            build_flags: '-include "^windows/amd64" -cgo'
-            build_args: '-buildmode exe'
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: windows_386
-            os: windows-latest
-            go: '1.16.x'
-            gotags: cmount
-            goarch: '386'
-            cgo: '1'
-            build_flags: '-include "^windows/386" -cgo'
-            build_args: '-buildmode exe'
-            quicktest: true
-            deploy: true
-
-          - job_name: other_os
-            os: ubuntu-latest
-            go: '1.16.x'
-            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
-            compile_all: true
-            deploy: true
-
-          - job_name: go1.13
-            os: ubuntu-latest
-            go: '1.13.x'
-            quicktest: true
-
-          - job_name: go1.14
-            os: ubuntu-latest
-            go: '1.14.x'
-            quicktest: true
-            racequicktest: true
-
-          - job_name: go1.15
-            os: ubuntu-latest
-            go: '1.15.x'
-            quicktest: true
-            racequicktest: true
-
-    name: ${{ matrix.job_name }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          stable: 'false'
-          go-version: ${{ matrix.go }}
-
-      - name: Set environment variables
-        shell: bash
-        run: |
-          echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
-          echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
-          echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV
-          if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi
-          if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
-
-      - name: Install Libraries on Linux
-        shell: bash
-        run: |
-          sudo modprobe fuse
-          sudo chmod 666 /dev/fuse
-          sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse libfuse-dev rpm pkg-config
-        if: matrix.os == 'ubuntu-latest'
-
-      - name: Install Libraries on macOS
-        shell: bash
-        run: |
-          brew update
-          brew install --cask macfuse
-        if: matrix.os == 'macOS-latest'
-
-      - name: Install Libraries on Windows
-        shell: powershell
-        run: |
-          $ProgressPreference = 'SilentlyContinue'
-          choco install -y winfsp zip
-          echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          if ($env:GOARCH -eq "386") {
-            choco install -y mingw --forcex86 --force
-            echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          }
-          # Copy mingw32-make.exe to make.exe so the same command line
-          # can be used on Windows as on macOS and Linux
-          $path = (get-command mingw32-make.exe).Path
-          Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
-        if: matrix.os == 'windows-latest'
-
-      - name: Print Go version and environment
-        shell: bash
-        run: |
-          printf "Using go at: $(which go)\n"
-          printf "Go version: $(go version)\n"
-          printf "\n\nGo environment:\n\n"
-          go env
-          printf "\n\nRclone environment:\n\n"
-          make vars
-          printf "\n\nSystem environment:\n\n"
-          env
-
-      - name: Go module cache
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Build rclone
-        shell: bash
-        run: |
-          make
-
-      - name: Run tests
-        shell: bash
-        run: |
-          make quicktest
-        if: matrix.quicktest
-
-      - name: Race test
-        shell: bash
-        run: |
-          make racequicktest
-        if: matrix.racequicktest
-
-      - name: Code quality test
-        shell: bash
-        run: |
-          make build_dep
-          make check
-        if: matrix.check
-
-      - name: Compile all architectures test
-        shell: bash
-        run: |
-          make
-          make compile_all
-        if: matrix.compile_all
-
-      - name: Deploy built binaries
-        shell: bash
-        run: |
-          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
-          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
-          make ci_beta
-        env:
-          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-        # working-directory: '$(modulePath)'
-        # Deploy binaries if enabled in config && not a PR && not a fork
-        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
-
-  android:
-     timeout-minutes: 30
-     name: "android-all"
-     runs-on: ubuntu-latest
-     
-     steps:
-       - name: Checkout
-         uses: actions/checkout@v2
-         with:
-           fetch-depth: 0
-
-       # Upgrade together with NDK version
-       - name: Set up Go 1.14
-         uses: actions/setup-go@v1
-         with:
-           go-version: 1.14
-
-       # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
-       - name: Force NDK version
-         run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;21.4.7075529" | grep -v = || true
-
-       - name: Go module cache
-         uses: actions/cache@v2
-         with:
-           path: ~/go/pkg/mod
-           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-           restore-keys: |
-             ${{ runner.os }}-go-
-
-       - name: Set global environment variables
-         shell: bash
-         run: |
-           echo "VERSION=$(make version)" >> $GITHUB_ENV
-
-       - name: build native rclone
-         run: |
-           make
-
-       - name: arm-v7a Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=arm' >> $GITHUB_ENV
-           echo 'GOARM=7' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-       - name: arm-v7a build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
-
-       - name: arm64-v8a Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=arm64' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: arm64-v8a build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
-
-       - name: x86 Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=386' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: x86 build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
-
-       - name: x64 Set environment variables
-         shell: bash
-         run: |
-           echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
-           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-           echo 'GOOS=android' >> $GITHUB_ENV
-           echo 'GOARCH=amd64' >> $GITHUB_ENV
-           echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-           echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-       - name: x64 build
-         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
-
-       - name: Upload artifacts
-         run: |
-           make ci_upload
-         env:
-           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-         # Upload artifacts if not a PR && not a fork
-         if: github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_docker_image.yml

@@ -1,25 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-
-jobs:
-    build:
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v2
-              with:
-                fetch-depth: 0
-            - name: Build and publish image
-              uses: ilteoood/docker_buildx@1.1.0
-              with:
-                tag: beta
-                imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
-                publish: true
-                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
-                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -1,33 +0,0 @@
-name: Docker release build
-
-on:
-  release:
-    types: [published]
-
-jobs:
-    build:
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Checkout master
-              uses: actions/checkout@v2
-              with:
-                fetch-depth: 0
-            - name: Get actual patch version
-              id: actual_patch_version
-              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
-            - name: Get actual minor version
-              id: actual_minor_version
-              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
-            - name: Get actual major version
-              id: actual_major_version
-              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Build and publish image
-              uses: ilteoood/docker_buildx@1.1.0
-              with:
-                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
-                imageName: rclone/rclone
-                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
-                publish: true
-                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
-                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.gitignore

@@ -1,13 +1,7 @@
 *~
 _junk/
 rclone
-rclone.exe
 build
 docs/public
 rclone.iml
 .idea
-.history
-*.test
-*.log
-*.iml
-fuzz-build.zip

.golangci.yml

@@ -1,26 +0,0 @@
-# golangci-lint configuration options
-
-linters:
-  enable:
-    - deadcode
-    - errcheck
-    - goimports
-    - golint
-    - ineffassign
-    - structcheck
-    - varcheck
-    - govet
-    - unconvert
-    #- prealloc
-    #- maligned
-  disable-all: true
-
-issues:
-  # Enable some lints excluded by default
-  exclude-use-default: false
-
-  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
-
-  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
-  max-same-issues: 0

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.pkgr.yml

@@ -0,0 +1,2 @@
+default_dependencies: false
+cli: rclone

.travis.yml

@@ -0,0 +1,51 @@
+language: go
+sudo: required
+dist: trusty
+os:
+- linux
+go:
+- 1.7.x
+- 1.8.x
+- 1.9.x
+- 1.10.x
+- 1.11.x
+- tip
+before_install:
+- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
+- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
+install:
+- git fetch --unshallow --tags
+- make vars
+- make build_dep
+script:
+- make check
+- make quicktest
+- make compile_all
+env:
+  global:
+    - GOTAGS=cmount
+    - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
+    - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
+    - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
+addons:
+  apt:
+    packages:
+    - fuse
+    - libfuse-dev
+    - rpm
+    - pkg-config
+matrix:
+  allow_failures:
+  - go: tip
+  include:
+  - os: osx
+    go: 1.11.x
+    env: GOTAGS=""
+deploy:
+  provider: script
+  script: make travis_beta
+  skip_cleanup: true
+  on:
+    all_branches: true
+    go: 1.11.x
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -12,32 +12,31 @@ When filing an issue, please include the following information if
 possible as well as a description of the problem.  Make sure you test
 with the [latest beta of rclone](https://beta.rclone.org/):
 
-  * Rclone version (e.g. output from `rclone -V`)
-  * Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
-  * The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
-  * A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
+  * Rclone version (eg output from `rclone -V`)
+  * Which OS you are using and how many bits (eg Windows 7, 64 bit)
+  * The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
+  * A log of the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
     * if the log contains secrets then edit the file with a text editor first to obscure them
 
 ## Submitting a pull request ##
 
 If you find a bug that you'd like to fix, or a new feature that you'd
-like to implement then please submit a pull request via GitHub.
+like to implement then please submit a pull request via Github.
 
 If it is a big feature then make an issue first so it can be discussed.
 
 You'll need a Go environment set up with GOPATH set.  See [the Go
 getting started docs](https://golang.org/doc/install) for more info.
 
-First in your web browser press the fork button on [rclone's GitHub
-page](https://github.com/rclone/rclone).
+First in your web browser press the fork button on [rclone's Github
+page](https://github.com/ncw/rclone).
 
 Now in your terminal
 
-    git clone https://github.com/rclone/rclone.git
-    cd rclone
+    go get -u github.com/ncw/rclone
+    cd $GOPATH/src/github.com/ncw/rclone
     git remote rename origin upstream
     git remote add origin git@github.com:YOURUSER/rclone.git
-    go build
 
 Make a branch to add your new feature
 
@@ -49,7 +48,7 @@ When ready - run the unit tests for the code you changed
 
     go test -v
 
-Note that you may need to make a test remote, e.g. `TestSwift` for some
+Note that you may need to make a test remote, eg `TestSwift` for some
 of the unit tests.
 
 Note the top level Makefile targets
@@ -65,34 +64,22 @@ packages which you can install with
 
 Make sure you
 
-  * Add [documentation](#writing-documentation) for a new feature.
-  * Follow the [commit message guidelines](#commit-messages).
-  * Add [unit tests](#testing) for a new feature
+  * Add documentation for a new feature (see below for where)
+  * Add unit tests for a new feature
   * squash commits down to one per feature
-  * rebase to master with `git rebase master`
+  * rebase to master `git rebase master`
 
 When you are done with that
 
-    git push -u origin my-new-feature
+  git push origin my-new-feature
 
-Go to the GitHub website and click [Create pull
+Go to the Github website and click [Create pull
 request](https://help.github.com/articles/creating-a-pull-request/).
 
 You patch will get reviewed and you might get asked to fix some stuff.
 
-If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
-```
-git log # See how many commits you want to squash
-git reset --soft HEAD~2 # This squashes the 2 latest commits together.
-git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
-git commit # Add a new commit message.
-git push --force # Push the squashed commit to your GitHub repo.
-# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also recommends wizardzines.com
-```
-
-## CI for your fork ##
-
-rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
+If so, then make the changes in the same branch, squash the commits,
+rebase it to master then push it to Github with `--force`.
 
 ## Testing ##
 
@@ -100,7 +87,7 @@ rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
 
     go test -v ./...
-
+    
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
 storage systems by mocking all their interfaces, rclone unit tests can
@@ -116,30 +103,23 @@ are skipped if `TestDrive:` isn't defined.
     cd backend/drive
     go test -v
 
-You can then run the integration tests which test all of rclone's
-operations.  Normally these get run against the local file system,
+You can then run the integration tests which tests all of rclone's
+operations.  Normally these get run against the local filing system,
 but they can be run against any of the remotes.
 
     cd fs/sync
     go test -v -remote TestDrive:
-    go test -v -remote TestDrive: -fast-list
+    go test -v -remote TestDrive: -subdir
 
     cd fs/operations
     go test -v -remote TestDrive:
 
-If you want to use the integration test framework to run these tests
-altogether with an HTML report and test retries then from the
-project root:
-
-    go install github.com/rclone/rclone/fstest/test_all
-    test_all -backend drive
-
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
 
     make test
 
-This command is run daily on the integration test server. You can
+This command is run daily on the the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##
@@ -156,7 +136,6 @@ with modules beneath.
     * ...commands
   * docs - the documentation and website
     * content - adjust these docs only - everything else is autogenerated
-      * command - these are auto generated - edit the corresponding .go file
   * fs - main rclone definitions - minimal amount of code
     * accounting - bandwidth limiting and statistics
     * asyncreader - an io.Reader which reads ahead
@@ -166,12 +145,12 @@ with modules beneath.
     * fserrors - rclone specific error handling
     * fshttp - http handling for rclone
     * fspath - path handling for rclone
-    * hash - defines rclone's hash types and functions
+    * hash - defines rclones hash types and functions
     * list - list a remote
     * log - logging facilities
     * march - iterates directories in lock step
     * object - in memory Fs objects
-    * operations - primitives for sync, e.g. Copy, Move
+    * operations - primitives for sync, eg Copy, Move
     * sync - sync directories
     * walk - walk a directory
   * fstest - provides integration test framework
@@ -179,7 +158,7 @@ with modules beneath.
     * mockdir - mocks an fs.Directory
     * mockobject - mocks an fs.Object
     * test_all - Runs integration tests for everything
-  * graphics - the images used in the website, etc.
+  * graphics - the images used in the website etc
   * lib - libraries used by the backend
     * atexit - register functions to run when rclone exits
     * dircache - directory ID to name caching
@@ -187,33 +166,27 @@ with modules beneath.
     * pacer - retries with backoff and paces operations
     * readers - a selection of useful io.Readers
     * rest - a thin abstraction over net/http for REST
+  * vendor - 3rd party code managed by `go mod`
   * vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
 ## Writing Documentation ##
 
 If you are adding a new feature then please update the documentation.
 
-If you add a new general flag (not for a backend), then document it in
+If you add a new flag, then if it is a general flag, document it in
 `docs/content/docs.md` - the flags there are supposed to be in
-alphabetical order.
-
-If you add a new backend option/flag, then it should be documented in
-the source file in the `Help:` field.  The first line of this is used
-for the flag help, the remainder is shown to the user in `rclone
-config` and is added to the docs with `make backenddocs`.
+alphabetical order.  If it is a remote specific flag, then document it
+in `docs/content/remote.md`.
 
 The only documentation you need to edit are the `docs/content/*.md`
-files.  The `MANUAL.*`, `rclone.1`, web site, etc. are all auto generated
+files.  The MANUAL.*, rclone.1, web site etc are all auto generated
 from those during the release process.  See the `make doc` and `make
 website` targets in the Makefile if you are interested in how.  You
 don't need to run these when adding a feature.
 
-Documentation for rclone sub commands is with their code, e.g.
+Documentation for rclone sub commands is with their code, eg
 `cmd/ls/ls.go`.
 
-Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
-for small changes in the docs which makes it very easy.
-
 ## Making a release ##
 
 There are separate instructions for making a release in the RELEASE.md
@@ -222,20 +195,14 @@ file.
 ## Commit messages ##
 
 Please make the first line of your commit message a summary of the
-change that a user (not a developer) of rclone would like to read, and
-prefix it with the directory of the change followed by a colon.  The
-changelog gets made by looking at just these first lines so make it
-good!
+change, and prefix it with the directory of the change followed by a
+colon.  The changelog gets made by looking at just these first lines
+so make it good!
 
 If you have more to say about the commit, then enter a blank line and
 carry on the description.  Remember to say why the change was needed -
 the commit itself shows what was changed.
 
-Writing more is better than less.  Comparing the behaviour before the
-change to that after the change is very useful.  Imagine you are
-writing to yourself in 12 months time when you've forgotten everything
-about what you just did and you need to get up to speed quickly.
-
 If the change fixes an issue then write `Fixes #1234` in the commit
 message.  This can be on the subject line if it will fit.  If you
 don't want to close the associated issue just put `#1234` and the
@@ -266,27 +233,45 @@ rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-rclone can be built with modules outside of the `GOPATH`.
+**NB** you must be using go1.11 or above to add a dependency to
+rclone.  Rclone will still build with older versions of go, but we use
+the `go mod` command for dependencies which is only in go1.11 and
+above.
+
+rclone can be built with modules outside of the GOPATH, but for
+backwards compatibility with older go versions, rclone also maintains
+a `vendor` directory with all the external code rclone needs for
+building.
+
+The `vendor` directory is entirely managed by the `go mod` tool, do
+not add things manually.
 
 To add a dependency `github.com/ncw/new_dependency` see the
-instructions below.  These will fetch the dependency and add it to
-`go.mod` and `go.sum`.
+instructions below.  These will fetch the dependency, add it to
+`go.mod` and `go.sum` and vendor it for older go versions.
 
-    GO111MODULE=on go get github.com/ncw/new_dependency
+    export GO111MODULE=on
+    go get github.com/ncw/new_dependency
+    go mod vendor
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
 
-Please check in the changes generated by `go mod` including `go.mod`
-and `go.sum` in the same commit as your other changes.
+Please check in the changes generated by `go mod` including the
+`vendor` directory and `go.mod` and `go.sum` in a single commit
+separate from any other code changes with the title "vendor: add
+github.com/ncw/new_dependency".  Remember to `git add` any new files
+in `vendor`.
 
 ## Updating a dependency ##
 
 If you need to update a dependency then run
 
-    GO111MODULE=on go get -u github.com/pkg/errors
+    export GO111MODULE=on
+    go get -u github.com/pkg/errors
+    go mod vendor
 
-Check in a single commit as above.
+Check in in a single commit as above.
 
 ## Updating all the dependencies ##
 
@@ -332,11 +317,6 @@ Getting going
   * Add your remote to the imports in `backend/all/all.go`
   * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
   * Try to implement as many optional methods as possible as it makes the remote more usable.
-  * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
-    * `rclone purge -v TestRemote:rclone-info`
-    * `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    * `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
-    * open `remote.csv` in a spreadsheet and examine
 
 Unit tests
 
@@ -346,71 +326,26 @@ Unit tests
 
 Integration tests
 
-  * Add your backend to `fstest/test_all/config.yaml`
-      * Once you've done that then you can use the integration test framework from the project root:
-      * go install ./...
-      * test_all -backends remote
-
-Or if you want to run the integration tests manually:
-
+  * Add your fs to `fstest/test_all/test_all.go`
   * Make sure integration tests pass with
       * `cd fs/operations`
       * `go test -v -remote TestRemote:`
       * `cd fs/sync`
       * `go test -v -remote TestRemote:`
-  * If your remote defines `ListR` check with this also
+  * If you are making a bucket based remote, then check with this also
+      * `go test -v -remote TestRemote: -subdir`
+  * And if your remote defines `ListR` this also
       * `go test -v -remote TestRemote: -fast-list`
 
 See the [testing](#testing) section for more information on integration tests.
 
-Add your fs to the docs - you'll need to pick an icon for it from
-[fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
-alphabetical order of full name of remote (e.g. `drive` is ordered as
-`Google Drive`) but with the local file system last.
+Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in alphabetical order but with the local file system last.
 
-  * `README.md` - main GitHub page
-  * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
-    * make sure this has the `autogenerated options` comments in (see your reference backend docs)
-    * update them with `make backenddocs` - revert any changes in other backends
+  * `README.md` - main Github page
+  * `docs/content/remote.md` - main docs page
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
-  * `docs/content/_index.md` - front page of rclone.org
+  * `docs/content/about.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
-
-Once you've written the docs, run `make serve` and check they look OK
-in the web browser and the links (internal and external) all work.
-
-## Writing a plugin ##
-
-New features (backends, commands) can also be added "out-of-tree", through Go plugins.
-Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
-This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
-
-Usage
-
- - Naming
-   - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
-   - `KIND` should be one of `backend`, `command` or `bundle`.
-   - Example: A plugin with backend support for PiFS would be called
-     `librcloneplugin_backend_pifs.so`.
- - Loading
-   - Supported on macOS & Linux as of now. ([Go issue for Windows support](https://github.com/golang/go/issues/19282))
-   - Supported on rclone v1.50 or greater.
-   - All plugins in the folder specified by variable `$RCLONE_PLUGIN_PATH` are loaded.
-   - If this variable doesn't exist, plugin support is disabled.
-   - Plugins must be compiled against the exact version of rclone to work.
-     (The rclone used during building the plugin must be the same as the source of rclone)
-
-Building
-
-To turn your existing additions into a Go plugin, move them to an external repository
-and change the top-level package name to `main`.
-
-Check `rclone --version` and make sure that the plugin's rclone dependency and host Go version match.
-
-Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
-
-[Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
-
-[Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)
+  * `cmd/cmd.go` - the main help for rclone

Dockerfile

@@ -1,24 +0,0 @@
-FROM golang AS builder
-
-COPY . /go/src/github.com/rclone/rclone/
-WORKDIR /go/src/github.com/rclone/rclone/
-
-RUN \
-  CGO_ENABLED=0 \
-  make
-RUN ./rclone version
-
-# Begin final image
-FROM alpine:latest
-
-RUN apk --no-cache add ca-certificates fuse tzdata && \
-  echo "user_allow_other" >> /etc/fuse.conf
-
-COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
-
-RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
-
-ENTRYPOINT [ "rclone" ]
-
-WORKDIR /data
-ENV XDG_CONFIG_HOME=/config

ISSUE_TEMPLATE.md

@@ -0,0 +1,43 @@
+<!--
+
+Hi!
+
+We understand you are having a problem with rclone or have an idea for an improvement - we want to help you with that!
+
+If you've just got a question or aren't sure if you've found a bug then please use the rclone forum
+
+    https://forum.rclone.org/
+
+instead of filing an issue.  We'll reply quickly and it won't increase our massive issue backlog.
+
+If you think you might have found a bug, please can you try to replicate it with the latest beta?
+
+    https://beta.rclone.org/
+    
+If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
+
+If you have an idea for an improvement, then please search the old issues first and if you don't find your idea, make a new issue.
+
+Thanks
+
+The Rclone Developers
+
+-->
+
+#### What is the problem you are having with rclone?
+
+
+#### What is your rclone version (eg output from `rclone -V`)
+
+
+#### Which OS you are using and how many bits (eg Windows 7, 64 bit)
+
+
+####  Which cloud storage system are you using? (eg Google Drive)
+
+
+#### The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
+
+
+#### A log from the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
+

MAINTAINERS.md

@@ -1,21 +1,12 @@
 # Maintainers guide for rclone #
 
-Current active maintainers of rclone are:
+Current active maintainers of rclone are
 
-| Name             | GitHub ID         | Specific Responsibilities    |
-| :--------------- | :---------------- | :--------------------------  |
-| Nick Craig-Wood  | @ncw              | overall project health       |
-| Stefan Breunig   | @breunigs         |                              |
-| Ishuah Kariuki   | @ishuah           |                              |
-| Remus Bunduc     | @remusb           | cache backend                |
-| Fabian Möller    | @B4dM4n           |                              |
-| Alex Chen        | @Cnly             | onedrive backend             |
-| Sandeep Ummadi   | @sandeepkru       | azureblob backend            |
-| Sebastian Bünger | @buengese         | jottacloud, yandex & compress backends |
-| Ivan Andreev     | @ivandeex         | chunker & mailru backends    |
-| Max Sum          | @Max-Sum          | union backend                |
-| Fred             | @creativeprojects | seafile backend              |
-| Caleb Case       | @calebcase        | tardigrade backend           |
+  * Nick Craig-Wood @ncw
+  * Stefan Breunig @breunigs
+  * Ishuah Kariuki @ishuah
+  * Remus Bunduc @remusb - cache subsystem maintainer
+  * Fabian Möller @B4dM4n
 
 **This is a work in progress Draft**
 
@@ -33,11 +24,11 @@ Rclone uses the labels like this:
 * `duplicate` - normally close these and ask the user to subscribe to the original
 * `enhancement: new remote` - a new rclone backend
 * `enhancement` - a new feature
-* `FUSE` - to do with `rclone mount` command
+* `FUSE` - do do with `rclone mount` command
 * `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
 * `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
 * `IMPORTANT` - note to maintainers not to forget to fix this for the release
-* `maintenance` - internal enhancement, code re-organisation, etc.
+* `maintenance` - internal enhancement, code re-organisation etc
 * `Needs Go 1.XX` - waiting for that version of Go to be released
 * `question` - not a `bug` or `enhancement` - direct to the forum for next time
 * `Remote: XXX` - which rclone backend this affects
@@ -45,7 +36,7 @@ Rclone uses the labels like this:
 
 If it turns out to be a bug or an enhancement it should be tagged as such, with the appropriate other tags.  Don't forget the "good first issue" tag to give new contributors something easy to do to get going.
 
-When a ticket is tagged it should be added to a milestone, either the next release, the one after, Soon or Help Wanted.  Bugs can be added to the "Known Bugs" milestone if they aren't planned to be fixed or need to wait for something (e.g. the next go release).
+When a ticket is tagged it should be added to a milestone, either the next release, the one after, Soon or Help Wanted.  Bugs can be added to the "Known Bugs" milestone if they aren't planned to be fixed or need to wait for something (eg the next go release).
 
 The milestones have these meanings:
 
@@ -55,7 +46,7 @@ The milestones have these meanings:
 * Help wanted - blue sky stuff that might get moved up, or someone could help with
 * Known bugs - bugs waiting on external factors or we aren't going to fix for the moment
 
-Tickets [with no milestone](https://github.com/rclone/rclone/issues?utf8=✓&q=is%3Aissue%20is%3Aopen%20no%3Amile) are good candidates for ones that have slipped between the gaps and need following up.
+Tickets [with no milestone](https://github.com/ncw/rclone/issues?utf8=✓&q=is%3Aissue%20is%3Aopen%20no%3Amile) are good candidates for ones that have slipped between the gaps and need following up.
 
 ## Closing Tickets ##
 
@@ -65,7 +56,7 @@ Close tickets as soon as you can - make sure they are tagged with a release.  Po
 
 Try to process pull requests promptly!
 
-Merging pull requests on GitHub itself works quite well now-a-days so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
+Merging pull requests on Github itself works quite well now-a-days so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
 
 After merging the commit, in your local master branch, do `git pull` then run `bin/update-authors.py` to update the authors file then `git push`.
 

Makefile

@@ -1,138 +1,123 @@
 SHELL = bash
-# Branch we are working on
-BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
-# Tag of the current commit, if any.  If this is not "" then we are building a release
-RELEASE_TAG := $(shell git tag -l --points-at HEAD)
-# Version of last release (may not be on this branch)
-VERSION := $(shell cat VERSION)
-# Last tag on this branch
+BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-# Next version
-NEXT_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2+1,0}')
-NEXT_PATCH_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2,$$3+1}')
-# If we are working on a release, override branch to master
-ifdef RELEASE_TAG
+ifeq ($(BRANCH),$(LAST_TAG))
 	BRANCH := master
-	LAST_TAG := $(shell git describe --abbrev=0 --tags $(VERSION)^)
 endif
-TAG_BRANCH := .$(BRANCH)
-BRANCH_PATH := branch/$(BRANCH)/
-# If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
+TAG_BRANCH := -$(BRANCH)
+BRANCH_PATH := branch/
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-# Make version suffix -beta.NNNN.CCCCCCCC (N=Commit number, C=Commit)
-VERSION_SUFFIX := -beta.$(shell git rev-list --count HEAD).$(shell git show --no-patch --no-notes --pretty='%h' HEAD)
-# TAG is current version + commit number + commit + branch
-TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
-ifdef RELEASE_TAG
-	TAG := $(RELEASE_TAG)
+TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
+ifneq ($(TAG),$(LAST_TAG))
+	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
-ifdef BETA_SUBDIR
-	BETA_SUBDIR := /$(BETA_SUBDIR)
-endif
-BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
+GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+# Run full tests if go >= go1.11
+FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
+BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
-LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
-.PHONY: rclone test_all vars version
+.PHONY: rclone vars version
 
 rclone:
-	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
-	mkdir -p `go env GOPATH`/bin/
-	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
-	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
-
-test_all:
-	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
+	touch fs/version.go
+	go install -v --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
+	cp -av `go env GOPATH`/bin/rclone .
 
 vars:
 	@echo SHELL="'$(SHELL)'"
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
-	@echo VERSION="'$(VERSION)'"
+	@echo LAST_TAG="'$(LAST_TAG)'"
+	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
+	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
-btest:
-	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
-	@echo "Copied markdown of beta release to clip board"
-
 version:
 	@echo '$(TAG)'
 
 # Full suite of integration tests
-test:	rclone test_all
-	-test_all 2>&1 | tee test_all.log
-	@echo "Written logs in test_all.log"
+test:	rclone
+	go install github.com/ncw/rclone/fstest/test_all
+	-go test -v -count 1 $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log
+	-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log
+	@echo "Written logs in test.log and fs/test_all.log"
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./...
-
-racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
+ifdef FULL_TESTS
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
+endif
 
 # Do source code quality checks
 check:	rclone
-	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@golangci-lint run $(LINTTAGS) ./...
-	@echo "-- END CODE QUALITY REPORT ---------------------------------"
+ifdef FULL_TESTS
+	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	errcheck $(BUILDTAGS) ./...
+	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
+	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+else
+	@echo Skipping source quality tests as version of go too old
+endif
+
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+ifdef FULL_TESTS
+	go get -u github.com/kisielk/errcheck
+	go get -u golang.org/x/tools/cmd/goimports
+	go get -u github.com/golang/lint/golint
+endif
 
-# Get the release dependencies we only install on linux
-release_dep_linux:
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64\.tar\.gz'
-
-# Get the release dependencies we only install on Windows
-release_dep_windows:
-	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
+# Get the release dependencies
+release_dep:
+	go get -u github.com/goreleaser/nfpm/...
+	go get -u github.com/aktau/github-release
 
 # Update dependencies
-showupdates:
-	@echo "*** Direct dependencies that could be updated ***"
-	@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
-
-# Update direct and indirect dependencies and test dependencies
 update:
-	GO111MODULE=on go get -u -t ./...
-	-#GO111MODULE=on go get -d $(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
-	GO111MODULE=on go mod tidy
-
-# Tidy the module dependencies
-tidy:
-	GO111MODULE=on go mod tidy
+	GO111MODULE=on go get -u ./...
+	GO111MODULE=on go tidy
+	GO111MODULE=on go vendor
 
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
-
-backenddocs: rclone bin/make_backend_docs.py
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
+	rclone gendocs docs/content/commands/
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -148,27 +133,14 @@ clean:
 	rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
 
 website:
-	rm -rf docs/public
 	cd docs && hugo
-	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
 	rclone -v sync docs/public memstore:www-rclone-org
 
-upload_test_website:	website
-	rclone -P sync docs/public test-rclone-org:
-
-validate_website: website
-	find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
-
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
-vendorball:
-	go mod vendor
-	tar -zcf build/rclone-$(TAG)-vendor.tar.gz vendor
-	rm -rf vendor
-
 sign_upload:
 	cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
 	cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
@@ -180,17 +152,17 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
+	rclone -v copy --exclude '*current*' build/ memstore:downloads-rclone-org/$(TAG)
+	rclone -v copy --include '*current*' --include version.txt build/ memstore:downloads-rclone-org
 
 upload_github:
 	./bin/upload-github $(TAG)
 
 cross:	doc
-	go run bin/cross-compile.go -release current $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -release current $(BUILDTAGS) $(TAG)
 
 beta:
-	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go $(BUILDTAGS) $(TAG)
 	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
@@ -198,61 +170,63 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -compile-only $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-
-ci_upload:
-	sudo chown -R $$USER build
-	find build -type l -delete
-	gzip -r9v build
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
-ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+ifdef FULL_TESTS
+	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+else
+	@echo Skipping compile all as version of go too old
 endif
-	@echo Beta release ready at $(BETA_URL)/testbuilds
 
-ci_beta:
-	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-# Fetch the binary builds from GitHub actions
+BUILD_FLAGS := -exclude "^(windows|darwin)/"
+ifeq ($(TRAVIS_OS_NAME),osx)
+	BUILD_FLAGS := -include "^darwin/" -cgo
+endif
+
+travis_beta:
+ifeq ($(TRAVIS_OS_NAME),linux)
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+endif
+	git log $(LAST_TAG).. > /tmp/git-log.txt
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
+endif
+	@echo Beta release ready at $(BETA_URL)
+
+# Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -v sync $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server -v -w --disableFastRender
+	cd docs && hugo server -v -w
 
-tag:	retag doc
-	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new
+tag:	doc
+	@echo "Old tag is $(LAST_TAG)"
+	@echo "New tag is $(NEW_TAG)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
+	echo -n "$(NEW_TAG)" > docs/layouts/partials/version.html
+	git tag -s -m "Version $(NEW_TAG)" $(NEW_TAG)
+	bin/make_changelog.py $(LAST_TAG) $(NEW_TAG) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(VERSION)\" -a -v
-	@echo "And finally run make retag before make cross, etc."
+	@echo git commit -m \"Version $(NEW_TAG)\" -a -v
+	@echo "And finally run make retag before make cross etc"
 
 retag:
-	@echo "Version is $(VERSION)"
-	git tag -f -s -m "Version $(VERSION)" $(VERSION)
+	git tag -f -s -m "Version $(LAST_TAG)" $(LAST_TAG)
 
 startdev:
-	@echo "Version is $(VERSION)"
-	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
-
-startstable:
-	@echo "Version is $(VERSION)"
-	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
-	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go
+	git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
+

README.md

@@ -1,115 +1,62 @@
-[<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/)
+[![Logo](https://rclone.org/img/rclone-120x120.png)](https://rclone.org/)
 
 [Website](https://rclone.org) |
 [Documentation](https://rclone.org/docs/) |
-[Download](https://rclone.org/downloads/) | 
 [Contributing](CONTRIBUTING.md) |
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/)
+[G+](https://google.com/+RcloneOrg)
 
-[![Build Status](https://github.com/rclone/rclone/workflows/build/badge.svg)](https://github.com/rclone/rclone/actions?query=workflow%3Abuild)
-[![Go Report Card](https://goreportcard.com/badge/github.com/rclone/rclone)](https://goreportcard.com/report/github.com/rclone/rclone)
-[![GoDoc](https://godoc.org/github.com/rclone/rclone?status.svg)](https://godoc.org/github.com/rclone/rclone) 
-[![Docker Pulls](https://img.shields.io/docker/pulls/rclone/rclone)](https://hub.docker.com/r/rclone/rclone)
+[![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
+[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
+[![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
+[![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
-# Rclone
+Rclone is a command line program to sync files and directories to and from
 
-Rclone *("rsync for cloud storage")* is a command line program to sync files and directories to and from different cloud storage providers.
+  * Amazon Drive ([See note](https://rclone.org/amazonclouddrive/#status))
+  * Amazon S3 / Dreamhost / Ceph / Minio / Wasabi
+  * Backblaze B2
+  * Box
+  * Dropbox
+  * FTP
+  * Google Cloud Storage
+  * Google Drive
+  * HTTP
+  * Hubic
+  * Jottacloud
+  * Mega
+  * Microsoft Azure Blob Storage
+  * Microsoft OneDrive
+  * OpenDrive
+  * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
+  * pCloud
+  * QingStor
+  * SFTP
+  * Webdav / Owncloud / Nextcloud
+  * Yandex Disk
+  * The local filesystem
 
-## Storage providers
+Features
 
-  * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
-  * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
-  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
-  * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
-  * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
-  * Box [:page_facing_up:](https://rclone.org/box/)
-  * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
-  * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
-  * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
-  * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
-  * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
-  * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
-  * FTP [:page_facing_up:](https://rclone.org/ftp/)
-  * GetSky [:page_facing_up:](https://rclone.org/jottacloud/)
-  * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
-  * Google Drive [:page_facing_up:](https://rclone.org/drive/)
-  * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
-  * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
-  * HTTP [:page_facing_up:](https://rclone.org/http/)
-  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
-  * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
-  * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
-  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
-  * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
-  * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
-  * Mega [:page_facing_up:](https://rclone.org/mega/)
-  * Memory [:page_facing_up:](https://rclone.org/memory/)
-  * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
-  * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
-  * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
-  * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
-  * OVH [:page_facing_up:](https://rclone.org/swift/)
-  * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
-  * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
-  * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
-  * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
-  * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
-  * put.io [:page_facing_up:](https://rclone.org/putio/)
-  * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
-  * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
-  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
-  * Seafile [:page_facing_up:](https://rclone.org/seafile/)
-  * SFTP [:page_facing_up:](https://rclone.org/sftp/)
-  * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
-  * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
-  * Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
-  * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
-  * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
-  * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
-  * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
-  * Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
-  * The local filesystem [:page_facing_up:](https://rclone.org/local/)
-  
-Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
-
-## Features
-
-  * MD5/SHA-1 hashes checked at all times for file integrity
+  * MD5/SHA1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
-  * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
-  * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
-  * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, e.g. two different cloud accounts
-  * Optional large file chunking ([Chunker](https://rclone.org/chunker/))
-  * Optional transparent compression ([Compress](https://rclone.org/compress/))
-  * Optional encryption ([Crypt](https://rclone.org/crypt/))
-  * Optional cache ([Cache](https://rclone.org/cache/))
-  * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
-  * Multi-threaded downloads to local disk
-  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
+  * Copy mode to just copy new/changed files
+  * Sync (one way) mode to make a directory identical
+  * Check mode to check for file hash equality
+  * Can sync to and from network, eg two different cloud accounts
+  * Optional encryption (Crypt)
+  * Optional FUSE mount
 
-## Installation & documentation
+See the home page for installation, usage, documentation, changelog
+and configuration walkthroughs.
 
-Please see the [rclone website](https://rclone.org/) for:
-
-  * [Installation](https://rclone.org/install/)
-  * [Documentation & configuration](https://rclone.org/docs/)
-  * [Changelog](https://rclone.org/changelog/)
-  * [FAQ](https://rclone.org/faq/)
-  * [Storage providers](https://rclone.org/overview/)
-  * [Forum](https://forum.rclone.org/)
-  * ...and more
-
-## Downloads
-
-  * https://rclone.org/downloads/
+  * https://rclone.org/
 
 License
 -------
 
 This is free software under the terms of MIT the license (check the
-[COPYING file](/COPYING) included in this package).
+COPYING file included in this package).

RELEASE.md

@@ -1,104 +1,33 @@
-# Release
-
-This file describes how to make the various kinds of releases
-
-## Extra required software for making a release
-
-  * [gh the github cli](https://github.com/cli/cli) for uploading packages
+Extra required software for making a release
+  * [github-release](https://github.com/aktau/github-release) for uploading packages
   * pandoc for making the html and man pages
 
-## Making a release
-
-  * git checkout master # see below for stable branch
-  * git pull
+Making a release
   * git status - make sure everything is checked in
-  * Check GitHub actions build for master is Green
+  * Check travis & appveyor builds are green
+  * make check
   * make test # see integration test server or run locally
   * make tag
-  * edit docs/content/changelog.md # make sure to remove duplicate logs from point releases
-  * make tidy
+  * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX.0"
+  * git commit -a -v -m "Version v1.XX"
   * make retag
-  * git push --follow-tags origin
-  * # Wait for the GitHub builds to complete then...
+  * git push --tags origin master
+  * # Wait for the appveyor and travis builds to complete then...
   * make fetch_binaries
   * make tarball
-  * make vendorball
   * make sign_upload
   * make check_sign
   * make upload
   * make upload_website
   * make upload_github
-  * make startdev # make startstable for stable branch
-  * # announce with forum post, twitter post, patreon post
-
-Early in the next release cycle update the dependencies
+  * make startdev
+  * # announce with forum post, twitter post, G+ post
 
+Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
   * make update
   * git status
   * git add new files
   * git commit -a -v
-
-## Making a point release
-
-If rclone needs a point release due to some horrendous bug:
-
-Set vars
-
-  * BASE_TAG=v1.XX          # e.g. v1.52
-  * NEW_TAG=${BASE_TAG}.Y   # e.g. v1.52.1
-  * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
-
-First make the release branch.  If this is a second point release then
-this will be done already.
-
-  * git branch ${BASE_TAG} ${BASE_TAG}-stable
-  * git co ${BASE_TAG}-stable
-  * make startstable
-
-Now
-
-  * git co ${BASE_TAG}-stable
-  * git cherry-pick any fixes
-  * Do the steps as above
-  * make startstable
-  * git co master
-  * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
-  * git checkout ${BASE_TAG}-stable docs/content/changelog.md
-  * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
-  * git push
-
-## Making a manual build of docker
-
-The rclone docker image should autobuild on via GitHub actions.  If it doesn't
-or needs to be updated then rebuild like this.
-
-See: https://github.com/ilteoood/docker_buildx/issues/19
-See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
-
-```
-git co v1.54.1
-docker pull golang
-export DOCKER_CLI_EXPERIMENTAL=enabled
-docker buildx create --name actions_builder --use
-docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
-echo "Supported platforms: $SUPPORTED_PLATFORMS"
-docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
-docker buildx stop actions_builder
-```
-
-### Old build for linux/amd64 only
-
-```
-docker pull golang
-docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
-docker push rclone/rclone:1.52.0
-docker push rclone/rclone:1.52
-docker push rclone/rclone:1
-docker push rclone/rclone:latest
-```

VERSION

@@ -1 +0,0 @@
-v1.56.0

backend/alias/alias.go

@@ -1,22 +1,21 @@
 package alias
 
 import (
-	"context"
 	"errors"
+	"path"
+	"path/filepath"
 	"strings"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fspath"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
 )
 
 // Register with Fs
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "alias",
-		Description: "Alias for an existing remote",
+		Description: "Alias for a existing remote",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "remote",
@@ -32,10 +31,10 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs constructs an Fs from the path.
+// NewFs contstructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -48,5 +47,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	return cache.Get(ctx, fspath.JoinRootPath(opt.Remote, root))
+	_, configName, fsPath, err := fs.ParseRemote(opt.Remote)
+	if err != nil {
+		return nil, err
+	}
+	root = path.Join(fsPath, filepath.ToSlash(root))
+	if configName == "local" {
+		return fs.NewFs(root)
+	}
+	return fs.NewFs(configName + ":" + root)
 }

backend/alias/alias_internal_test.go

@@ -1,17 +1,15 @@
 package alias
 
 import (
-	"context"
 	"fmt"
 	"path"
 	"path/filepath"
 	"sort"
 	"testing"
 
-	_ "github.com/rclone/rclone/backend/local" // pull in test backend
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configfile"
+	_ "github.com/ncw/rclone/backend/local" // pull in test backend
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,7 +18,7 @@ var (
 )
 
 func prepare(t *testing.T, root string) {
-	require.NoError(t, configfile.LoadConfig(context.Background()))
+	config.LoadConfig()
 
 	// Configure the remote
 	config.FileSet(remoteName, "type", "alias")
@@ -55,24 +53,23 @@ func TestNewFS(t *testing.T) {
 			{"four/under four.txt", 9, false},
 		}},
 		{"four", "..", "", true, []testEntry{
-			{"five", -1, true},
-			{"under four.txt", 9, false},
+			{"four", -1, true},
+			{"one%.txt", 6, false},
+			{"three", -1, true},
+			{"two.html", 7, false},
 		}},
-		{"", "../../three", "", true, []testEntry{
+		{"four", "../three", "", true, []testEntry{
 			{"underthree.txt", 9, false},
 		}},
-		{"four", "../../five", "", true, []testEntry{
-			{"underfive.txt", 6, false},
-		}},
 	} {
 		what := fmt.Sprintf("test %d remoteRoot=%q, fsRoot=%q, fsList=%q", testi, test.remoteRoot, test.fsRoot, test.fsList)
 
 		remoteRoot, err := filepath.Abs(filepath.FromSlash(path.Join("test/files", test.remoteRoot)))
 		require.NoError(t, err, what)
 		prepare(t, remoteRoot)
-		f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
+		f, err := fs.NewFs(fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
 		require.NoError(t, err, what)
-		gotEntries, err := f.List(context.Background(), test.fsList)
+		gotEntries, err := f.List(test.fsList)
 		require.NoError(t, err, what)
 
 		sort.Sort(gotEntries)
@@ -83,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
@@ -92,7 +89,7 @@ func TestNewFS(t *testing.T) {
 
 func TestNewFSNoRemote(t *testing.T) {
 	prepare(t, "")
-	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)
@@ -100,7 +97,7 @@ func TestNewFSNoRemote(t *testing.T) {
 
 func TestNewFSInvalidRemote(t *testing.T) {
 	prepare(t, "not_existing_test_remote:")
-	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)

backend/all/all.go

@@ -2,47 +2,29 @@ package all
 
 import (
 	// Active file systems
-	_ "github.com/rclone/rclone/backend/alias"
-	_ "github.com/rclone/rclone/backend/amazonclouddrive"
-	_ "github.com/rclone/rclone/backend/azureblob"
-	_ "github.com/rclone/rclone/backend/b2"
-	_ "github.com/rclone/rclone/backend/box"
-	_ "github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/compress"
-	_ "github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive"
-	_ "github.com/rclone/rclone/backend/dropbox"
-	_ "github.com/rclone/rclone/backend/fichier"
-	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/ftp"
-	_ "github.com/rclone/rclone/backend/googlecloudstorage"
-	_ "github.com/rclone/rclone/backend/googlephotos"
-	_ "github.com/rclone/rclone/backend/hdfs"
-	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/hubic"
-	_ "github.com/rclone/rclone/backend/jottacloud"
-	_ "github.com/rclone/rclone/backend/koofr"
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/mailru"
-	_ "github.com/rclone/rclone/backend/mega"
-	_ "github.com/rclone/rclone/backend/memory"
-	_ "github.com/rclone/rclone/backend/onedrive"
-	_ "github.com/rclone/rclone/backend/opendrive"
-	_ "github.com/rclone/rclone/backend/pcloud"
-	_ "github.com/rclone/rclone/backend/premiumizeme"
-	_ "github.com/rclone/rclone/backend/putio"
-	_ "github.com/rclone/rclone/backend/qingstor"
-	_ "github.com/rclone/rclone/backend/s3"
-	_ "github.com/rclone/rclone/backend/seafile"
-	_ "github.com/rclone/rclone/backend/sftp"
-	_ "github.com/rclone/rclone/backend/sharefile"
-	_ "github.com/rclone/rclone/backend/sugarsync"
-	_ "github.com/rclone/rclone/backend/swift"
-	_ "github.com/rclone/rclone/backend/tardigrade"
-	_ "github.com/rclone/rclone/backend/union"
-	_ "github.com/rclone/rclone/backend/uptobox"
-	_ "github.com/rclone/rclone/backend/webdav"
-	_ "github.com/rclone/rclone/backend/yandex"
-	_ "github.com/rclone/rclone/backend/zoho"
+	_ "github.com/ncw/rclone/backend/alias"
+	_ "github.com/ncw/rclone/backend/amazonclouddrive"
+	_ "github.com/ncw/rclone/backend/azureblob"
+	_ "github.com/ncw/rclone/backend/b2"
+	_ "github.com/ncw/rclone/backend/box"
+	_ "github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/crypt"
+	_ "github.com/ncw/rclone/backend/drive"
+	_ "github.com/ncw/rclone/backend/dropbox"
+	_ "github.com/ncw/rclone/backend/ftp"
+	_ "github.com/ncw/rclone/backend/googlecloudstorage"
+	_ "github.com/ncw/rclone/backend/http"
+	_ "github.com/ncw/rclone/backend/hubic"
+	_ "github.com/ncw/rclone/backend/jottacloud"
+	_ "github.com/ncw/rclone/backend/local"
+	_ "github.com/ncw/rclone/backend/mega"
+	_ "github.com/ncw/rclone/backend/onedrive"
+	_ "github.com/ncw/rclone/backend/opendrive"
+	_ "github.com/ncw/rclone/backend/pcloud"
+	_ "github.com/ncw/rclone/backend/qingstor"
+	_ "github.com/ncw/rclone/backend/s3"
+	_ "github.com/ncw/rclone/backend/sftp"
+	_ "github.com/ncw/rclone/backend/swift"
+	_ "github.com/ncw/rclone/backend/webdav"
+	_ "github.com/ncw/rclone/backend/yandex"
 )

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -7,9 +7,9 @@ package amazonclouddrive_test
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/amazonclouddrive"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/amazonclouddrive"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote

backend/azureblob/azureblob_internal_test.go

@@ -1,35 +0,0 @@
-// +build !plan9,!solaris,!js,go1.14
-
-package azureblob
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func (f *Fs) InternalTest(t *testing.T) {
-	// Check first feature flags are set on this
-	// remote
-	enabled := f.Features().SetTier
-	assert.True(t, enabled)
-	enabled = f.Features().GetTier
-	assert.True(t, enabled)
-}
-
-func TestIncrement(t *testing.T) {
-	for _, test := range []struct {
-		in   []byte
-		want []byte
-	}{
-		{[]byte{0, 0, 0, 0}, []byte{1, 0, 0, 0}},
-		{[]byte{0xFE, 0, 0, 0}, []byte{0xFF, 0, 0, 0}},
-		{[]byte{0xFF, 0, 0, 0}, []byte{0, 1, 0, 0}},
-		{[]byte{0, 1, 0, 0}, []byte{1, 1, 0, 0}},
-		{[]byte{0xFF, 0xFF, 0xFF, 0xFE}, []byte{0, 0, 0, 0xFF}},
-		{[]byte{0xFF, 0xFF, 0xFF, 0xFF}, []byte{0, 0, 0, 0}},
-	} {
-		increment(test.in)
-		assert.Equal(t, test.want, test.in)
-	}
-}

backend/azureblob/azureblob_test.go

@@ -1,64 +1,20 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris,!js,go1.14
+// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
 
-package azureblob
+package azureblob_test
 
 import (
-	"context"
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/stretchr/testify/assert"
+	"github.com/ncw/rclone/backend/azureblob"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
-		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MaxChunkSize: maxChunkSize,
-		},
+		RemoteName: "TestAzureBlob:",
+		NilObject:  (*azureblob.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-)
-
-// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
-func TestServicePrincipalFileSuccess(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "password": "my secret",
-    "tenant": "my active directory tenant ID"
-}
-`
-	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	if assert.NoError(t, err) {
-		assert.NotNil(t, tokenRefresher)
-	}
-}
-
-// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
-func TestServicePrincipalFileFailure(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "tenant": "my active directory tenant ID"
-}
-`
-	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
-	assert.Error(t, err)
-	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
-}

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris js !go1.14
+// +build freebsd netbsd openbsd plan9 solaris !go1.8
 
 package azureblob

backend/azureblob/imds.go

@@ -1,137 +0,0 @@
-// +build !plan9,!solaris,!js,go1.14
-
-package azureblob
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-)
-
-const (
-	azureResource      = "https://storage.azure.com"
-	imdsAPIVersion     = "2018-02-01"
-	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
-)
-
-// This custom type is used to add the port the test server has bound to
-// to the request context.
-type testPortKey string
-
-type msiIdentifierType int
-
-const (
-	msiClientID msiIdentifierType = iota
-	msiObjectID
-	msiResourceID
-)
-
-type userMSI struct {
-	Type  msiIdentifierType
-	Value string
-}
-
-type httpError struct {
-	Response *http.Response
-}
-
-func (e httpError) Error() string {
-	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
-}
-
-// GetMSIToken attempts to obtain an MSI token from the Azure Instance
-// Metadata Service.
-func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
-	// Attempt to get an MSI token; silently continue if unsuccessful.
-	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
-	result := adal.Token{}
-	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
-	if err != nil {
-		fs.Debugf(nil, "Failed to create request: %v", err)
-		return result, err
-	}
-	params := req.URL.Query()
-	params.Set("resource", azureResource)
-	params.Set("api-version", imdsAPIVersion)
-
-	// Specify user-assigned identity if requested.
-	if identity != nil {
-		switch identity.Type {
-		case msiClientID:
-			params.Set("client_id", identity.Value)
-		case msiObjectID:
-			params.Set("object_id", identity.Value)
-		case msiResourceID:
-			params.Set("mi_res_id", identity.Value)
-		default:
-			// If this happens, the calling function and this one don't agree on
-			// what valid ID types exist.
-			return result, fmt.Errorf("unknown MSI identity type specified")
-		}
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// The Metadata header is required by all calls to IMDS.
-	req.Header.Set("Metadata", "true")
-
-	// If this function is run in a test, query the test server instead of IMDS.
-	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
-	if isTest {
-		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
-		req.Host = req.URL.Host
-	}
-
-	// Send request
-	httpClient := fshttp.NewClient(ctx)
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return result, errors.Wrap(err, "MSI is not enabled on this VM")
-	}
-	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
-		if err != nil {
-			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
-		}
-		err = resp.Body.Close()
-		if err != nil {
-			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
-		}
-	}()
-	// Check if the status code indicates success
-	// The request returns 200 currently, add 201 and 202 as well for possible extension.
-	switch resp.StatusCode {
-	case 200, 201, 202:
-		break
-	default:
-		body, _ := ioutil.ReadAll(resp.Body)
-		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
-		return result, httpError{Response: resp}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return result, errors.Wrap(err, "Couldn't read IMDS response")
-	}
-	// Remove BOM, if any. azcopy does this so I'm following along.
-	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
-
-	// This would be a good place to persist the token if a large number of rclone
-	// invocations are being made in a short amount of time. If the token is
-	// persisted, the azureblob code will need to check for expiry before every
-	// storage API call.
-	err = json.Unmarshal(b, &result)
-	if err != nil {
-		return result, errors.Wrap(err, "Couldn't unmarshal IMDS response")
-	}
-
-	return result, nil
-}

backend/azureblob/imds_test.go

@@ -1,117 +0,0 @@
-// +build !plan9,!solaris,!js,go1.14
-
-package azureblob
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		err := r.ParseForm()
-		require.NoError(t, err)
-		parameters := r.URL.Query()
-		(*actual)["path"] = r.URL.Path
-		(*actual)["Metadata"] = r.Header.Get("Metadata")
-		(*actual)["method"] = r.Method
-		for paramName := range parameters {
-			(*actual)[paramName] = parameters.Get(paramName)
-		}
-		// Make response.
-		response := adal.Token{}
-		responseBytes, err := json.Marshal(response)
-		require.NoError(t, err)
-		_, err = w.Write(responseBytes)
-		require.NoError(t, err)
-	}
-}
-
-func TestManagedIdentity(t *testing.T) {
-	// test user-assigned identity specifiers to use
-	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
-	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
-	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
-	tests := []struct {
-		identity              *userMSI
-		identityParameterName string
-		expectedAbsent        []string
-	}{
-		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
-		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
-		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
-		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
-	}
-	alwaysExpected := map[string]string{
-		"path":        "/metadata/identity/oauth2/token",
-		"resource":    "https://storage.azure.com",
-		"Metadata":    "true",
-		"api-version": "2018-02-01",
-		"method":      "GET",
-	}
-	for _, test := range tests {
-		actual := make(map[string]string, 10)
-		testServer := httptest.NewServer(handler(t, &actual))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, test.identity)
-		require.NoError(t, err)
-
-		// Validate expected query parameters present
-		expected := make(map[string]string)
-		for k, v := range alwaysExpected {
-			expected[k] = v
-		}
-		if test.identity != nil {
-			expected[test.identityParameterName] = test.identity.Value
-		}
-
-		for key := range expected {
-			value, exists := actual[key]
-			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
-				test.identityParameterName, key) {
-				assert.Equalf(t, expected[key], value,
-					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
-			}
-		}
-
-		// Validate unexpected query parameters absent
-		for _, key := range test.expectedAbsent {
-			_, exists := actual[key]
-			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
-		}
-	}
-}
-
-func errorHandler(resultCode int) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Test error generated", resultCode)
-	}
-}
-
-func TestIMDSErrors(t *testing.T) {
-	errorCodes := []int{404, 429, 500}
-	for _, code := range errorCodes {
-		testServer := httptest.NewServer(errorHandler(code))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, nil)
-		require.Error(t, err)
-		httpErr, ok := err.(httpError)
-		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
-		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
-	}
-}

backend/b2/api/types.go

@@ -2,11 +2,12 @@ package api
 
 import (
 	"fmt"
+	"path"
 	"strconv"
+	"strings"
 	"time"
 
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/version"
+	"github.com/ncw/rclone/fs/fserrors"
 )
 
 // Error describes a B2 error response
@@ -16,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error satisfies the error interface
+// Error statisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal satisfies the Fatal interface
+// Fatal statisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -49,7 +50,7 @@ type Timestamp time.Time
 // MarshalJSON turns a Timestamp into JSON (in UTC)
 func (t *Timestamp) MarshalJSON() (out []byte, err error) {
 	timestamp := (*time.Time)(t).UTC().UnixNano()
-	return []byte(strconv.FormatInt(timestamp/1e6, 10)), nil
+	return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil
 }
 
 // UnmarshalJSON turns JSON into a Timestamp
@@ -58,21 +59,20 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		return err
 	}
-	*t = Timestamp(time.Unix(timestamp/1e3, (timestamp%1e3)*1e6).UTC())
+	*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6).UTC())
 	return nil
 }
 
-// HasVersion returns true if it looks like the passed filename has a timestamp on it.
-//
-// Note that the passed filename's timestamp may still be invalid even if this
-// function returns true.
-func HasVersion(remote string) bool {
-	return version.Match(remote)
-}
+const versionFormat = "-v2006-01-02-150405.000"
 
 // AddVersion adds the timestamp as a version string into the filename passed in.
 func (t Timestamp) AddVersion(remote string) string {
-	return version.Add(remote, time.Time(t))
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	s := time.Time(t).Format(versionFormat)
+	// Replace the '.' with a '-'
+	s = strings.Replace(s, ".", "-", -1)
+	return base + s + ext
 }
 
 // RemoveVersion removes the timestamp from a filename as a version string.
@@ -80,12 +80,27 @@ func (t Timestamp) AddVersion(remote string) string {
 // It returns the new file name and a timestamp, or the old filename
 // and a zero timestamp.
 func RemoveVersion(remote string) (t Timestamp, newRemote string) {
-	time, newRemote := version.Remove(remote)
-	t = Timestamp(time)
-	return
+	newRemote = remote
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	if len(base) < len(versionFormat) {
+		return
+	}
+	versionStart := len(base) - len(versionFormat)
+	// Check it ends in -xxx
+	if base[len(base)-4] != '-' {
+		return
+	}
+	// Replace with .xxx for parsing
+	base = base[:len(base)-4] + "." + base[len(base)-3:]
+	newT, err := time.Parse(versionFormat, base[versionStart:])
+	if err != nil {
+		return
+	}
+	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is uninitialized
+// IsZero returns true if the timestamp is unitialised
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }
@@ -121,7 +136,6 @@ type AuthorizeAccountResponse struct {
 	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
 		BucketID     string      `json:"bucketId"`     // When present, access is restricted to one bucket.
-		BucketName   string      `json:"bucketName"`   // When present, name of bucket - may be empty
 		Capabilities []string    `json:"capabilities"` // A list of strings, each one naming a capability the key has.
 		NamePrefix   interface{} `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`
@@ -174,21 +188,6 @@ type GetUploadURLResponse struct {
 	AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file.
 }
 
-// GetDownloadAuthorizationRequest is passed to b2_get_download_authorization
-type GetDownloadAuthorizationRequest struct {
-	BucketID               string `json:"bucketId"`                       // The ID of the bucket that you want to upload to.
-	FileNamePrefix         string `json:"fileNamePrefix"`                 // The file name prefix of files the download authorization token will allow access to.
-	ValidDurationInSeconds int64  `json:"validDurationInSeconds"`         // The number of seconds before the authorization token will expire. The minimum value is 1 second. The maximum value is 604800 which is one week in seconds.
-	B2ContentDisposition   string `json:"b2ContentDisposition,omitempty"` // optional - If this is present, download requests using the returned authorization must include the same value for b2ContentDisposition.
-}
-
-// GetDownloadAuthorizationResponse is received from b2_get_download_authorization
-type GetDownloadAuthorizationResponse struct {
-	BucketID           string `json:"bucketId"`           // The unique ID of the bucket.
-	FileNamePrefix     string `json:"fileNamePrefix"`     // The file name prefix of files the download authorization token will allow access to.
-	AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when downloading files, see b2_download_file_by_name.
-}
-
 // FileInfo is received from b2_upload_file, b2_get_file_info and b2_finish_large_file
 type FileInfo struct {
 	ID              string            `json:"fileId"`          // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
@@ -311,22 +310,3 @@ type CancelLargeFileResponse struct {
 	AccountID string `json:"accountId"` // The identifier for the account.
 	BucketID  string `json:"bucketId"`  // The unique ID of the bucket.
 }
-
-// CopyFileRequest is as passed to b2_copy_file
-type CopyFileRequest struct {
-	SourceID          string            `json:"sourceFileId"`                  // The ID of the source file being copied.
-	Name              string            `json:"fileName"`                      // The name of the new file being created.
-	Range             string            `json:"range,omitempty"`               // The range of bytes to copy. If not provided, the whole source file will be copied.
-	MetadataDirective string            `json:"metadataDirective,omitempty"`   // The strategy for how to populate metadata for the new file: COPY or REPLACE
-	ContentType       string            `json:"contentType,omitempty"`         // The MIME type of the content of the file (REPLACE only)
-	Info              map[string]string `json:"fileInfo,omitempty"`            // This field stores the metadata that will be stored with the file. (REPLACE only)
-	DestBucketID      string            `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
-}
-
-// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
-type CopyPartRequest struct {
-	SourceID    string `json:"sourceFileId"`    // The ID of the source file being copied.
-	LargeFileID string `json:"largeFileId"`     // The ID of the large file the part will belong to, as returned by b2_start_large_file.
-	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
-	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
-}

backend/b2/api/types_test.go

@@ -4,8 +4,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fstest"
+	"github.com/ncw/rclone/backend/b2/api"
+	"github.com/ncw/rclone/fstest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -13,6 +13,7 @@ import (
 var (
 	emptyT api.Timestamp
 	t0     = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
+	t0r    = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
 	t1     = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
 )
 
@@ -35,6 +36,40 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
 	assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
 }
 
+func TestTimestampAddVersion(t *testing.T) {
+	for _, test := range []struct {
+		t        api.Timestamp
+		in       string
+		expected string
+	}{
+		{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
+		{t1, "potato", "potato-v2001-02-03-040506-123"},
+		{t1, "", "-v2001-02-03-040506-123"},
+	} {
+		actual := test.t.AddVersion(test.in)
+		assert.Equal(t, test.expected, actual, test.in)
+	}
+}
+
+func TestTimestampRemoveVersion(t *testing.T) {
+	for _, test := range []struct {
+		in             string
+		expectedT      api.Timestamp
+		expectedRemote string
+	}{
+		{"potato.txt", emptyT, "potato.txt"},
+		{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
+		{"potato-v2001-02-03-040506-123", t1, "potato"},
+		{"-v2001-02-03-040506-123", t1, ""},
+		{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
+		{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
+	} {
+		actualT, actualRemote := api.RemoveVersion(test.in)
+		assert.Equal(t, test.expectedT, actualT, test.in)
+		assert.Equal(t, test.expectedRemote, actualRemote, test.in)
+	}
+}
+
 func TestTimestampIsZero(t *testing.T) {
 	assert.True(t, emptyT.IsZero())
 	assert.False(t, t0.IsZero())

backend/b2/b2_internal_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/fstest"
+	"github.com/ncw/rclone/fstest"
 )
 
 // Test b2 string encoding

backend/b2/b2_test.go

@@ -1,34 +1,17 @@
 // Test B2 filesystem interface
-package b2
+package b2_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/b2"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestB2:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize:       minChunkSize,
-			NeedMultipleChunks: true,
-		},
+		NilObject:  (*b2.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/b2/upload.go

@@ -6,7 +6,6 @@ package b2
 
 import (
 	"bytes"
-	"context"
 	"crypto/sha1"
 	"encoding/hex"
 	"fmt"
@@ -15,14 +14,12 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/ncw/rclone/backend/b2/api"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/sync/errgroup"
 )
 
 type hashAppendingReader struct {
@@ -70,26 +67,20 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
 
 // largeUpload is used to control the upload of large files which need chunking
 type largeUpload struct {
-	f         *Fs                             // parent Fs
-	o         *Object                         // object being uploaded
-	doCopy    bool                            // doing copy rather than upload
-	what      string                          // text name of operation for logs
-	in        io.Reader                       // read the data from here
-	wrap      accounting.WrapFn               // account parts being transferred
-	id        string                          // ID of the file being uploaded
-	size      int64                           // total size
-	parts     int64                           // calculated number of parts, if known
-	sha1s     []string                        // slice of SHA1s for each part
-	uploadMu  sync.Mutex                      // lock for upload variable
-	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
-	chunkSize int64                           // chunk size to use
-	src       *Object                         // if copying, object we are reading from
+	f        *Fs                             // parent Fs
+	o        *Object                         // object being uploaded
+	in       io.Reader                       // read the data from here
+	wrap     accounting.WrapFn               // account parts being transferred
+	id       string                          // ID of the file being uploaded
+	size     int64                           // total size
+	parts    int64                           // calculated number of parts, if known
+	sha1s    []string                        // slice of SHA1s for each part
+	uploadMu sync.Mutex                      // lock for upload variable
+	uploads  []*api.GetUploadPartURLResponse // result of get upload URL calls
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
-//
-// If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
+func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
 	remote := o.remote
 	size := src.Size()
 	parts := int64(0)
@@ -97,8 +88,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		parts = size / int64(chunkSize)
-		if size%int64(chunkSize) != 0 {
+		parts = size / int64(o.fs.opt.ChunkSize)
+		if size%int64(o.fs.opt.ChunkSize) != 0 {
 			parts++
 		}
 		if parts > maxParts {
@@ -107,61 +98,47 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		sha1SliceSize = parts
 	}
 
+	modTime := src.ModTime()
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_start_large_file",
 	}
-	bucket, bucketPath := o.split()
-	bucketID, err := f.getBucketID(ctx, bucket)
+	bucketID, err := f.getBucketID()
 	if err != nil {
 		return nil, err
 	}
 	var request = api.StartLargeFileRequest{
-		BucketID: bucketID,
-		Name:     f.opt.Enc.FromStandardPath(bucketPath),
-	}
-	if newInfo == nil {
-		modTime := src.ModTime(ctx)
-		request.ContentType = fs.MimeType(ctx, src)
-		request.Info = map[string]string{
+		BucketID:    bucketID,
+		Name:        o.fs.root + remote,
+		ContentType: fs.MimeType(src),
+		Info: map[string]string{
 			timeKey: timeString(modTime),
-		}
-		// Set the SHA1 if known
-		if !o.fs.opt.DisableCheckSum || doCopy {
-			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
-				request.Info[sha1Key] = calculatedSha1
-			}
-		}
-	} else {
-		request.ContentType = newInfo.ContentType
-		request.Info = newInfo.Info
+		},
+	}
+	// Set the SHA1 if known
+	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+		request.Info[sha1Key] = calculatedSha1
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
-		return f.shouldRetry(ctx, resp, err)
+		resp, err := f.srv.CallJSON(&opts, &request, &response)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
 	}
-	up = &largeUpload{
-		f:         f,
-		o:         o,
-		doCopy:    doCopy,
-		what:      "upload",
-		id:        response.ID,
-		size:      size,
-		parts:     parts,
-		sha1s:     make([]string, sha1SliceSize),
-		chunkSize: int64(chunkSize),
-	}
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
-	if doCopy {
-		up.what = "copy"
-		up.src = src.(*Object)
-	} else {
-		up.in, up.wrap = accounting.UnWrap(in)
+	in, wrap := accounting.UnWrap(in)
+	up = &largeUpload{
+		f:     f,
+		o:     o,
+		in:    in,
+		wrap:  wrap,
+		id:    response.ID,
+		size:  size,
+		parts: parts,
+		sha1s: make([]string, sha1SliceSize),
 	}
 	return up, nil
 }
@@ -169,7 +146,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
 //
 // This should be returned with returnUploadURL when finished
-func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
+func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) {
 	up.uploadMu.Lock()
 	defer up.uploadMu.Unlock()
 	if len(up.uploads) == 0 {
@@ -181,8 +158,8 @@ func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadP
 			ID: up.id,
 		}
 		err := up.f.pacer.Call(func() (bool, error) {
-			resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
-			return up.f.shouldRetry(ctx, resp, err)
+			resp, err := up.f.srv.CallJSON(&opts, &request, &upload)
+			return up.f.shouldRetry(resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to get upload URL")
@@ -203,13 +180,20 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
+// clearUploadURL clears the current UploadURL and the AuthorizationToken
+func (up *largeUpload) clearUploadURL() {
+	up.uploadMu.Lock()
+	up.uploads = nil
+	up.uploadMu.Unlock()
+}
+
 // Transfer a chunk
-func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
+func (up *largeUpload) transferChunk(part int64, body []byte) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
 
 		// Get upload URL
-		upload, err := up.getUploadURL(ctx)
+		upload, err := up.getUploadURL()
 		if err != nil {
 			return false, err
 		}
@@ -253,8 +237,8 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 
 		var response api.UploadPartResponse
 
-		resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
-		retry, err := up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, nil, &response)
+		retry, err := up.f.shouldRetry(resp, err)
 		if err != nil {
 			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
@@ -275,41 +259,9 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 	return err
 }
 
-// Copy a chunk
-func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
-	err := up.f.pacer.Call(func() (bool, error) {
-		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/b2_copy_part",
-		}
-		offset := (part - 1) * up.chunkSize // where we are in the source file
-		var request = api.CopyPartRequest{
-			SourceID:    up.src.id,
-			LargeFileID: up.id,
-			PartNumber:  part,
-			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
-		}
-		var response api.UploadPartResponse
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		retry, err := up.f.shouldRetry(ctx, resp, err)
-		if err != nil {
-			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
-		}
-		up.sha1s[part-1] = response.SHA1
-		return retry, err
-	})
-	if err != nil {
-		fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
-	} else {
-		fs.Debugf(up.o, "Done copying chunk %d", part)
-	}
-	return err
-}
-
 // finish closes off the large upload
-func (up *largeUpload) finish(ctx context.Context) error {
-	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
+func (up *largeUpload) finish() error {
+	fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_finish_large_file",
@@ -320,8 +272,8 @@ func (up *largeUpload) finish(ctx context.Context) error {
 	}
 	var response api.FileInfo
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		return up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
+		return up.f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
@@ -330,8 +282,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
 }
 
 // cancel aborts the large upload
-func (up *largeUpload) cancel(ctx context.Context) error {
-	fs.Debugf(up.o, "Cancelling large file %s", up.what)
+func (up *largeUpload) cancel() error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_cancel_large_file",
@@ -341,142 +292,142 @@ func (up *largeUpload) cancel(ctx context.Context) error {
 	}
 	var response api.CancelLargeFileResponse
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		return up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
+		return up.f.shouldRetry(resp, err)
 	})
-	if err != nil {
-		fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
-	}
 	return err
 }
 
+func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
+	wg.Add(1)
+	go func(part int64, buf []byte) {
+		defer wg.Done()
+		defer up.f.putUploadBlock(buf)
+		err := up.transferChunk(part, buf)
+		if err != nil {
+			select {
+			case errs <- err:
+			default:
+			}
+		}
+	}(part, buf)
+}
+
+func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error {
+	if err == nil {
+		select {
+		case err = <-errs:
+		default:
+		}
+	}
+	if err != nil {
+		fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
+		cancelErr := up.cancel()
+		if cancelErr != nil {
+			fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
+		}
+		return err
+	}
+	return up.finish()
+}
+
 // Stream uploads the chunks from the input, starting with a required initial
 // chunk. Assumes the file size is unknown and will upload until the input
 // reaches EOF.
-//
-// Note that initialUploadBlock must be returned to f.putBuf()
-func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) {
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
-	var (
-		g, gCtx      = errgroup.WithContext(ctx)
-		hasMoreParts = true
-	)
+	errs := make(chan error, 1)
+	hasMoreParts := true
+	var wg sync.WaitGroup
+
+	// Transfer initial chunk
 	up.size = int64(len(initialUploadBlock))
-	g.Go(func() error {
-		for part := int64(1); hasMoreParts; part++ {
-			// Get a block of memory from the pool and token which limits concurrency.
-			var buf []byte
-			if part == 1 {
-				buf = initialUploadBlock
-			} else {
-				buf = up.f.getBuf(false)
-			}
+	up.managedTransferChunk(&wg, errs, 1, initialUploadBlock)
 
-			// Fail fast, in case an errgroup managed function returns an error
-			// gCtx is cancelled. There is no point in uploading all the other parts.
-			if gCtx.Err() != nil {
-				up.f.putBuf(buf, false)
-				return nil
-			}
-
-			// Read the chunk
-			var n int
-			if part == 1 {
-				n = len(buf)
-			} else {
-				n, err = io.ReadFull(up.in, buf)
-				if err == io.ErrUnexpectedEOF {
-					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
-					buf = buf[:n]
-					hasMoreParts = false
-				} else if err == io.EOF {
-					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
-					up.f.putBuf(buf, false)
-					return nil
-				} else if err != nil {
-					// other kinds of errors indicate failure
-					up.f.putBuf(buf, false)
-					return err
-				}
-			}
-
-			// Keep stats up to date
-			up.parts = part
-			up.size += int64(n)
-			if part > maxParts {
-				up.f.putBuf(buf, false)
-				return errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
-			}
-
-			part := part // for the closure
-			g.Go(func() (err error) {
-				defer up.f.putBuf(buf, false)
-				return up.transferChunk(gCtx, part, buf)
-			})
+outer:
+	for part := int64(2); hasMoreParts; part++ {
+		// Check any errors
+		select {
+		case err = <-errs:
+			break outer
+		default:
 		}
-		return nil
-	})
-	err = g.Wait()
-	if err != nil {
-		return err
+
+		// Get a block of memory
+		buf := up.f.getUploadBlock()
+
+		// Read the chunk
+		var n int
+		n, err = io.ReadFull(up.in, buf)
+		if err == io.ErrUnexpectedEOF {
+			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+			buf = buf[:n]
+			hasMoreParts = false
+			err = nil
+		} else if err == io.EOF {
+			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+			up.f.putUploadBlock(buf)
+			err = nil
+			break outer
+		} else if err != nil {
+			// other kinds of errors indicate failure
+			up.f.putUploadBlock(buf)
+			break outer
+		}
+
+		// Keep stats up to date
+		up.parts = part
+		up.size += int64(n)
+		if part > maxParts {
+			err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			break outer
+		}
+
+		// Transfer the chunk
+		up.managedTransferChunk(&wg, errs, part, buf)
 	}
+	wg.Wait()
 	up.sha1s = up.sha1s[:up.parts]
-	return up.finish(ctx)
+
+	return up.finishOrCancelOnError(err, errs)
 }
 
 // Upload uploads the chunks from the input
-func (up *largeUpload) Upload(ctx context.Context) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
-	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
-	var (
-		g, gCtx   = errgroup.WithContext(ctx)
-		remaining = up.size
-	)
-	g.Go(func() error {
-		for part := int64(1); part <= up.parts; part++ {
-			// Get a block of memory from the pool and token which limits concurrency.
-			buf := up.f.getBuf(up.doCopy)
-
-			// Fail fast, in case an errgroup managed function returns an error
-			// gCtx is cancelled. There is no point in uploading all the other parts.
-			if gCtx.Err() != nil {
-				up.f.putBuf(buf, up.doCopy)
-				return nil
-			}
-
-			reqSize := remaining
-			if reqSize >= up.chunkSize {
-				reqSize = up.chunkSize
-			}
-
-			if !up.doCopy {
-				// Read the chunk
-				buf = buf[:reqSize]
-				_, err = io.ReadFull(up.in, buf)
-				if err != nil {
-					up.f.putBuf(buf, up.doCopy)
-					return err
-				}
-			}
-
-			part := part // for the closure
-			g.Go(func() (err error) {
-				defer up.f.putBuf(buf, up.doCopy)
-				if !up.doCopy {
-					err = up.transferChunk(gCtx, part, buf)
-				} else {
-					err = up.copyChunk(gCtx, part, reqSize)
-				}
-				return err
-			})
-			remaining -= reqSize
+func (up *largeUpload) Upload() error {
+	fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
+	remaining := up.size
+	errs := make(chan error, 1)
+	var wg sync.WaitGroup
+	var err error
+outer:
+	for part := int64(1); part <= up.parts; part++ {
+		// Check any errors
+		select {
+		case err = <-errs:
+			break outer
+		default:
 		}
-		return nil
-	})
-	err = g.Wait()
-	if err != nil {
-		return err
+
+		reqSize := remaining
+		if reqSize >= int64(up.f.opt.ChunkSize) {
+			reqSize = int64(up.f.opt.ChunkSize)
+		}
+
+		// Get a block of memory
+		buf := up.f.getUploadBlock()[:reqSize]
+
+		// Read the chunk
+		_, err = io.ReadFull(up.in, buf)
+		if err != nil {
+			up.f.putUploadBlock(buf)
+			break outer
+		}
+
+		// Transfer the chunk
+		up.managedTransferChunk(&wg, errs, part, buf)
+		remaining -= reqSize
 	}
-	return up.finish(ctx)
+	wg.Wait()
+
+	return up.finishOrCancelOnError(err, errs)
 }

backend/box/api/types.go

@@ -36,16 +36,16 @@ func (t *Time) UnmarshalJSON(data []byte) error {
 
 // Error is returned from box when things go wrong
 type Error struct {
-	Type        string          `json:"type"`
-	Status      int             `json:"status"`
-	Code        string          `json:"code"`
-	ContextInfo json.RawMessage `json:"context_info"`
-	HelpURL     string          `json:"help_url"`
-	Message     string          `json:"message"`
-	RequestID   string          `json:"request_id"`
+	Type        string `json:"type"`
+	Status      int    `json:"status"`
+	Code        string `json:"code"`
+	ContextInfo json.RawMessage
+	HelpURL     string `json:"help_url"`
+	Message     string `json:"message"`
+	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,11 +57,11 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error satisfies the error interface
+// Check Error statisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo
-var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link"
+var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status"
 
 // Types of things in Item
 const (
@@ -86,10 +86,6 @@ type Item struct {
 	ContentCreatedAt  Time    `json:"content_created_at"`
 	ContentModifiedAt Time    `json:"content_modified_at"`
 	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
-	SharedLink        struct {
-		URL    string `json:"url,omitempty"`
-		Access string `json:"access,omitempty"`
-	} `json:"shared_link"`
 }
 
 // ModTime returns the modification time of the item
@@ -132,38 +128,6 @@ type UploadFile struct {
 	ContentModifiedAt Time   `json:"content_modified_at"`
 }
 
-// PreUploadCheck is the request for upload preflight check
-type PreUploadCheck struct {
-	Name   string `json:"name"`
-	Parent Parent `json:"parent"`
-	Size   *int64 `json:"size,omitempty"`
-}
-
-// PreUploadCheckResponse is the response from upload preflight check
-// if successful
-type PreUploadCheckResponse struct {
-	UploadToken string `json:"upload_token"`
-	UploadURL   string `json:"upload_url"`
-}
-
-// PreUploadCheckConflict is returned in the ContextInfo error field
-// from PreUploadCheck when the error code is "item_name_in_use"
-type PreUploadCheckConflict struct {
-	Conflicts struct {
-		Type        string `json:"type"`
-		ID          string `json:"id"`
-		FileVersion struct {
-			Type string `json:"type"`
-			ID   string `json:"id"`
-			Sha1 string `json:"sha1"`
-		} `json:"file_version"`
-		SequenceID string `json:"sequence_id"`
-		Etag       string `json:"etag"`
-		Sha1       string `json:"sha1"`
-		Name       string `json:"name"`
-	} `json:"conflicts"`
-}
-
 // UpdateFileModTime is used in Update File Info
 type UpdateFileModTime struct {
 	ContentModifiedAt Time `json:"content_modified_at"`
@@ -181,14 +145,6 @@ type CopyFile struct {
 	Parent Parent `json:"parent"`
 }
 
-// CreateSharedLink is the request for Public Link
-type CreateSharedLink struct {
-	SharedLink struct {
-		URL    string `json:"url,omitempty"`
-		Access string `json:"access,omitempty"`
-	} `json:"shared_link"`
-}
-
 // UploadSessionRequest is uses in Create Upload Session
 type UploadSessionRequest struct {
 	FolderID string `json:"folder_id,omitempty"` // don't pass for update
@@ -234,43 +190,3 @@ type CommitUpload struct {
 		ContentModifiedAt Time `json:"content_modified_at"`
 	} `json:"attributes"`
 }
-
-// ConfigJSON defines the shape of a box config.json
-type ConfigJSON struct {
-	BoxAppSettings AppSettings `json:"boxAppSettings"`
-	EnterpriseID   string      `json:"enterpriseID"`
-}
-
-// AppSettings defines the shape of the boxAppSettings within box config.json
-type AppSettings struct {
-	ClientID     string  `json:"clientID"`
-	ClientSecret string  `json:"clientSecret"`
-	AppAuth      AppAuth `json:"appAuth"`
-}
-
-// AppAuth defines the shape of the appAuth within boxAppSettings in config.json
-type AppAuth struct {
-	PublicKeyID string `json:"publicKeyID"`
-	PrivateKey  string `json:"privateKey"`
-	Passphrase  string `json:"passphrase"`
-}
-
-// User is returned from /users/me
-type User struct {
-	Type          string    `json:"type"`
-	ID            string    `json:"id"`
-	Name          string    `json:"name"`
-	Login         string    `json:"login"`
-	CreatedAt     time.Time `json:"created_at"`
-	ModifiedAt    time.Time `json:"modified_at"`
-	Language      string    `json:"language"`
-	Timezone      string    `json:"timezone"`
-	SpaceAmount   int64     `json:"space_amount"`
-	SpaceUsed     int64     `json:"space_used"`
-	MaxUploadSize int64     `json:"max_upload_size"`
-	Status        string    `json:"status"`
-	JobTitle      string    `json:"job_title"`
-	Phone         string    `json:"phone"`
-	Address       string    `json:"address"`
-	AvatarURL     string    `json:"avatar_url"`
-}

backend/box/box_test.go

@@ -4,8 +4,8 @@ package box_test
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/box"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/box"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote

backend/box/upload.go

@@ -1,10 +1,9 @@
-// multipart upload for box
+// multpart upload for box
 
 package box
 
 import (
 	"bytes"
-	"context"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/json"
@@ -15,16 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ncw/rclone/backend/box/api"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/backend/box/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/rest"
 )
 
 // createUploadSession creates an upload session for the object
-func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
+func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions",
@@ -39,12 +37,12 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	} else {
 		opts.Path = "/files/upload_sessions"
 		request.FolderID = directoryID
-		request.FileName = o.fs.opt.Enc.FromStandardName(leaf)
+		request.FileName = replaceReservedChars(leaf)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.CallJSON(&opts, &request, &response)
+		return shouldRetry(resp, err)
 	})
 	return
 }
@@ -55,7 +53,7 @@ func sha1Digest(digest []byte) string {
 }
 
 // uploadPart uploads a part in an upload session
-func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
+func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) {
 	chunkSize := int64(len(chunk))
 	sha1sum := sha1.Sum(chunk)
 	opts := rest.Opts{
@@ -65,7 +63,6 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 		ContentType:   "application/octet-stream",
 		ContentLength: &chunkSize,
 		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
-		Options:       options,
 		ExtraHeaders: map[string]string{
 			"Digest": sha1Digest(sha1sum[:]),
 		},
@@ -73,8 +70,8 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.CallJSON(&opts, nil, &response)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -83,7 +80,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 }
 
 // commitUpload finishes an upload session
-func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
+func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions/" + SessionID + "/commit",
@@ -100,22 +97,22 @@ func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api
 	var body []byte
 	var resp *http.Response
 	// For discussion of this value see:
-	// https://github.com/rclone/rclone/issues/2054
+	// https://github.com/ncw/rclone/issues/2054
 	maxTries := o.fs.opt.CommitRetries
 	const defaultDelay = 10
 	var tries int
 outer:
 	for tries = 0; tries < maxTries; tries++ {
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
+			resp, err = o.fs.srv.CallJSON(&opts, &request, nil)
 			if err != nil {
-				return shouldRetry(ctx, resp, err)
+				return shouldRetry(resp, err)
 			}
 			body, err = rest.ReadBody(resp)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		delay := defaultDelay
-		var why string
+		why := "unknown"
 		if err != nil {
 			// Sometimes we get 400 Error with
 			// parts_mismatch immediately after uploading
@@ -157,7 +154,7 @@ outer:
 }
 
 // abortUpload cancels an upload session
-func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) {
+func (o *Object) abortUpload(SessionID string) (err error) {
 	opts := rest.Opts{
 		Method:     "DELETE",
 		Path:       "/files/upload_sessions/" + SessionID,
@@ -166,16 +163,16 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.Call(&opts)
+		return shouldRetry(resp, err)
 	})
 	return err
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
+func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) {
 	// Create upload session
-	session, err := o.createUploadSession(ctx, leaf, directoryID, size)
+	session, err := o.createUploadSession(leaf, directoryID, size)
 	if err != nil {
 		return errors.Wrap(err, "multipart upload create session failed")
 	}
@@ -183,13 +180,15 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
 
 	// Cancel the session if something went wrong
-	defer atexit.OnError(&err, func() {
-		fs.Debugf(o, "Cancelling multipart upload: %v", err)
-		cancelErr := o.abortUpload(ctx, session.ID)
-		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+	defer func() {
+		if err != nil {
+			fs.Debugf(o, "Cancelling multipart upload: %v", err)
+			cancelErr := o.abortUpload(session.ID)
+			if cancelErr != nil {
+				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
+			}
 		}
-	})()
+	}()
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
@@ -212,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		if reqSize >= int64(chunkSize) {
+			reqSize = int64(chunkSize)
 		}
 
 		// Make a block of memory
@@ -236,7 +235,7 @@ outer:
 			defer wg.Done()
 			defer o.fs.uploadToken.Put()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
-			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
+			partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap)
 			if err != nil {
 				err = errors.Wrap(err, "multipart upload failed to upload part")
 				select {
@@ -264,7 +263,7 @@ outer:
 	}
 
 	// Finalise the upload session
-	result, err := o.commitUpload(ctx, session.ID, parts, modTime, hash.Sum(nil))
+	result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil))
 	if err != nil {
 		return errors.Wrap(err, "multipart upload failed to finalize")
 	}

backend/cache/cache_internal_test.go

@@ -1,14 +1,9 @@
-// +build !plan9,!js
-// +build !race
+// +build !plan9
 
 package cache_test
 
 import (
 	"bytes"
-	"context"
-	"encoding/base64"
-	goflag "flag"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -16,24 +11,35 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime/debug"
+	"runtime"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/backend/cache"
-	"github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive"
-	"github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/testy"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/vfs/vfsflags"
+
+	"encoding/base64"
+	goflag "flag"
+	"fmt"
+	"runtime/debug"
+
+	"encoding/json"
+	"net/http"
+
+	"github.com/ncw/rclone/backend/cache"
+	"github.com/ncw/rclone/backend/crypt"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/object"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/fs/rc/rcflags"
+	"github.com/ncw/rclone/fstest"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/stretchr/testify/require"
 )
 
@@ -49,7 +55,9 @@ const (
 
 var (
 	remoteName                  string
+	mountDir                    string
 	uploadDir                   string
+	useMount                    bool
 	runInstance                 *run
 	errNotSupported             = errors.New("not supported")
 	decryptedToEncryptedRemotes = map[string]string{
@@ -85,7 +93,9 @@ var (
 
 func init() {
 	goflag.StringVar(&remoteName, "remote-internal", "TestInternalCache", "Remote to test with, defaults to local filesystem")
+	goflag.StringVar(&mountDir, "mount-dir-internal", "", "")
 	goflag.StringVar(&uploadDir, "upload-dir-internal", "", "")
+	goflag.BoolVar(&useMount, "cache-use-mount", false, "Test only with mount")
 }
 
 // TestMain drives the tests
@@ -93,7 +103,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
-	log.Printf("Running with the following params: \n remote: %v", remoteName)
+	log.Printf("Running with the following params: \n remote: %v, \n mount: %v", remoteName, useMount)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -115,7 +125,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	require.NoError(t, err)
 	listRootInner, err := runInstance.list(t, rootFs, innerFolder)
 	require.NoError(t, err)
-	listInner, err := rootFs2.List(context.Background(), "")
+	listInner, err := rootFs2.List("")
 	require.NoError(t, err)
 
 	require.Len(t, listRoot, 1)
@@ -133,10 +143,10 @@ func TestInternalVfsCache(t *testing.T) {
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "test")
+	err := rootFs.Mkdir("test")
 	require.NoError(t, err)
 	runInstance.writeObjectString(t, rootFs, "test/second", "content")
-	_, err = rootFs.List(context.Background(), "test")
+	_, err = rootFs.List("test")
 	require.NoError(t, err)
 
 	testReader := runInstance.randomReader(t, testSize)
@@ -261,13 +271,37 @@ func TestInternalObjNotFound(t *testing.T) {
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	obj, err := rootFs.NewObject(context.Background(), "404")
+	obj, err := rootFs.NewObject("404")
 	require.Error(t, err)
 	require.Nil(t, obj)
 }
 
+func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
+	if !runInstance.useMount {
+		t.Skip("test needs mount mode")
+	}
+	id := fmt.Sprintf("tirwffim%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+
+	var testData []byte
+	if runInstance.rootIsCrypt {
+		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)
+		require.NoError(t, err)
+	} else {
+		testData = []byte("test content")
+	}
+
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test"), testData)
+	data, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, "test content", string(data))
+}
+
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
-	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -311,7 +345,6 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 }
 
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
-	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -326,8 +359,8 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 		testData2, err = base64.StdEncoding.DecodeString(cryptedText2Base64)
 		require.NoError(t, err)
 	} else {
-		testData1 = []byte(random.String(100))
-		testData2 = []byte(random.String(200))
+		testData1 = []byte(fstest.RandomString(100))
+		testData2 = []byte(fstest.RandomString(200))
 	}
 
 	// write the object
@@ -359,10 +392,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), testSize)
+	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -417,7 +450,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	require.NoError(t, err)
 	log.Printf("original size: %v", originalSize)
 
-	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	expectedSize := int64(len([]byte("test content")))
 	var data2 []byte
@@ -429,7 +462,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 		data2 = []byte("test content")
 	}
 	objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap())
-	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
+	err = o.Update(bytes.NewReader(data2), objInfo)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data2)), o.Size())
 	log.Printf("updated size: %v", len(data2))
@@ -475,9 +508,9 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	} else {
 		testData = []byte("test content")
 	}
-	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test"))
-	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/one"))
-	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/second"))
+	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test"))
+	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/one"))
+	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/second"))
 	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
 
 	// list in mount
@@ -487,7 +520,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	require.NoError(t, err)
 
 	// move file
-	_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
+	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
 	require.NoError(t, err)
 
 	err = runInstance.retryBlock(func() error {
@@ -561,9 +594,9 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	} else {
 		testData = []byte("test content")
 	}
-	err = rootFs.Mkdir(context.Background(), "test")
+	err = rootFs.Mkdir("test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "test/one")
+	err = rootFs.Mkdir("test/one")
 	require.NoError(t, err)
 	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
 
@@ -580,7 +613,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	require.False(t, found)
 
 	// move file
-	_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
+	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
 	require.NoError(t, err)
 
 	err = runInstance.retryBlock(func() error {
@@ -642,23 +675,101 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
-	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	wrappedTime := time.Now().Add(-1 * time.Hour)
-	err = o.SetModTime(context.Background(), wrappedTime)
+	err = o.SetModTime(wrappedTime)
 	require.NoError(t, err)
 
 	// get a new instance from the cache
-	co, err := rootFs.NewObject(context.Background(), "data.bin")
+	co, err := rootFs.NewObject("data.bin")
 	require.NoError(t, err)
-	require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
+	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
 
 	cfs.DirCacheFlush() // flush the cache
 
 	// get a new instance from the cache
-	co, err = rootFs.NewObject(context.Background(), "data.bin")
+	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
-	require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
+	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+}
+
+func TestInternalChangeSeenAfterRc(t *testing.T) {
+	rcflags.Opt.Enabled = true
+	rc.Start(&rcflags.Opt)
+
+	id := fmt.Sprintf("ticsarc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	if !runInstance.useMount {
+		t.Skipf("needs mount")
+	}
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("needs drive")
+	}
+
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+	chunkSize := cfs.ChunkSize()
+
+	// create some rand test data
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
+	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
+
+	// update in the wrapped fs
+	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	require.NoError(t, err)
+	wrappedTime := time.Now().Add(-1 * time.Hour)
+	err = o.SetModTime(wrappedTime)
+	require.NoError(t, err)
+
+	// get a new instance from the cache
+	co, err := rootFs.NewObject("data.bin")
+	require.NoError(t, err)
+	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
+
+	m := make(map[string]string)
+	res, err := http.Post(fmt.Sprintf("http://localhost:5572/cache/expire?remote=%s", "data.bin"), "application/json; charset=utf-8", strings.NewReader(""))
+	require.NoError(t, err)
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	_ = json.NewDecoder(res.Body).Decode(&m)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached file cleared")
+
+	// get a new instance from the cache
+	co, err = rootFs.NewObject("data.bin")
+	require.NoError(t, err)
+	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+	li1, err := runInstance.list(t, rootFs, "")
+
+	// create some rand test data
+	testData2 := randStringBytes(int(chunkSize))
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
+
+	// list should have 1 item only
+	li1, err = runInstance.list(t, rootFs, "")
+	require.Len(t, li1, 1)
+
+	m = make(map[string]string)
+	res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader(""))
+	require.NoError(t, err)
+	defer func() {
+		_ = res2.Body.Close()
+	}()
+	_ = json.NewDecoder(res2.Body).Decode(&m)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached directory cleared")
+
+	// list should have 2 items now
+	li2, err := runInstance.list(t, rootFs, "")
+	require.Len(t, li2, 2)
 }
 
 func TestInternalCacheWrites(t *testing.T) {
@@ -693,7 +804,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	// create some rand test data
 	testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
-	o, err := cfs.NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	co, ok := o.(*cache.Object)
 	require.True(t, ok)
@@ -732,7 +843,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, l, 1)
 
-	err = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/third"))
+	err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third"))
 	require.NoError(t, err)
 
 	l, err = runInstance.list(t, rootFs, "test")
@@ -767,14 +878,14 @@ func TestInternalBug2117(t *testing.T) {
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
-	err = cfs.UnWrap().Mkdir(context.Background(), "test")
+	err = cfs.UnWrap().Mkdir("test")
 	require.NoError(t, err)
 	for i := 1; i <= 4; i++ {
-		err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d", i))
+		err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i))
 		require.NoError(t, err)
 
 		for j := 1; j <= 4; j++ {
-			err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d/dir%d", i, j))
+			err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j))
 			require.NoError(t, err)
 
 			runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test")
@@ -808,9 +919,15 @@ func TestInternalBug2117(t *testing.T) {
 type run struct {
 	okDiff            time.Duration
 	runDefaultCfgMap  configmap.Simple
+	mntDir            string
 	tmpUploadDir      string
+	useMount          bool
+	isMounted         bool
 	rootIsCrypt       bool
 	wrappedIsExternal bool
+	unmountFn         func() error
+	unmountRes        chan error
+	vfs               *vfs.VFS
 	tempFiles         []*os.File
 	dbPath            string
 	chunkPath         string
@@ -820,7 +937,9 @@ type run struct {
 func newRun() *run {
 	var err error
 	r := &run{
-		okDiff: time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this
+		okDiff:    time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this
+		useMount:  useMount,
+		isMounted: false,
 	}
 
 	// Read in all the defaults for all the options
@@ -833,10 +952,36 @@ func newRun() *run {
 		r.runDefaultCfgMap.Set(option.Name, fmt.Sprint(option.Default))
 	}
 
+	if mountDir == "" {
+		if runtime.GOOS != "windows" {
+			r.mntDir, err = ioutil.TempDir("", "rclonecache-mount")
+			if err != nil {
+				log.Fatalf("Failed to create mount dir: %v", err)
+				return nil
+			}
+		} else {
+			// Find a free drive letter
+			drive := ""
+			for letter := 'E'; letter <= 'Z'; letter++ {
+				drive = string(letter) + ":"
+				_, err := os.Stat(drive + "\\")
+				if os.IsNotExist(err) {
+					goto found
+				}
+			}
+			log.Print("Couldn't find free drive letter for test")
+		found:
+			r.mntDir = drive
+		}
+	} else {
+		r.mntDir = mountDir
+	}
+	log.Printf("Mount Dir: %v", r.mntDir)
+
 	if uploadDir == "" {
 		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
 		if err != nil {
-			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
+			log.Fatalf("Failed to create temp dir: %v", err)
 		}
 	} else {
 		r.tmpUploadDir = uploadDir
@@ -872,15 +1017,6 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		return nil, nil
 	}
 
-	// Config to pass to NewFs
-	m := configmap.Simple{}
-	for k, v := range r.runDefaultCfgMap {
-		m.Set(k, v)
-	}
-	for k, v := range flags {
-		m.Set(k, v)
-	}
-
 	// if the remote doesn't exist, create a new one with a local one for it
 	// identify which is the cache remote (it can be wrapped by a crypt too)
 	rootIsCrypt := false
@@ -889,10 +1025,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		localRemote := remote + "-local"
 		config.FileSet(localRemote, "type", "local")
 		config.FileSet(localRemote, "nounc", "true")
-		m.Set("type", "cache")
-		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
+		config.FileSet(remote, "type", "cache")
+		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
 	} else {
-		remoteType := config.FileGet(remote, "type")
+		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -900,17 +1036,17 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		if remoteType != "cache" {
 			if remoteType == "crypt" {
 				rootIsCrypt = true
-				m.Set("password", cryptPassword1)
-				m.Set("password2", cryptPassword2)
+				config.FileSet(remote, "password", cryptPassword1)
+				config.FileSet(remote, "password2", cryptPassword2)
 			}
-			remoteRemote := config.FileGet(remote, "remote")
+			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.FileGet(remoteWrapping, "type")
+			remoteType := config.FileGet(remoteWrapping, "type", "")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -925,15 +1061,22 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	boltDb, err := cache.GetPersistent(runInstance.dbPath, runInstance.chunkPath, &cache.Features{PurgeDb: true})
 	require.NoError(t, err)
 
-	ci := fs.GetConfig(context.Background())
-	ci.LowLevelRetries = 1
+	fs.Config.LowLevelRetries = 1
+
+	m := configmap.Simple{}
+	for k, v := range r.runDefaultCfgMap {
+		m.Set(k, v)
+	}
+	for k, v := range flags {
+		m.Set(k, v)
+	}
 
 	// Instantiate root
 	if purge {
 		boltDb.PurgeTempUploads()
 		_ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id))
 	}
-	f, err := cache.NewFs(context.Background(), remote, id, m)
+	f, err := cache.NewFs(remote, id, m)
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
@@ -947,21 +1090,33 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = f.Features().Purge(context.Background(), "")
+		_ = f.Features().Purge()
 		require.NoError(t, err)
 	}
-	err = f.Mkdir(context.Background(), "")
+	err = f.Mkdir("")
 	require.NoError(t, err)
+	if r.useMount && !r.isMounted {
+		r.mountFs(t, f)
+	}
+
 	return f, boltDb
 }
 
 func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
-	err := f.Features().Purge(context.Background(), "")
+	if r.useMount && r.isMounted {
+		r.unmountFs(t, f)
+	}
+
+	err := f.Features().Purge()
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
 	cfs.StopBackgroundRunners()
 
+	if r.useMount && runtime.GOOS != "windows" {
+		err = os.RemoveAll(r.mntDir)
+		require.NoError(t, err)
+	}
 	err = os.RemoveAll(r.tmpUploadDir)
 	require.NoError(t, err)
 
@@ -992,6 +1147,23 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	return f
 }
 
+func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string {
+	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
+	// create some rand test data
+	testData := randStringBytes(int(size))
+
+	r.writeRemoteBytes(t, f, remote, testData)
+	return remote
+}
+
+func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object {
+	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
+	// create some rand test data
+	testData := randStringBytes(int(size))
+
+	return r.writeObjectBytes(t, f, remote, testData)
+}
+
 func (r *run) writeRemoteString(t *testing.T, f fs.Fs, remote, content string) {
 	r.writeRemoteBytes(t, f, remote, []byte(content))
 }
@@ -1001,17 +1173,43 @@ func (r *run) writeObjectString(t *testing.T, f fs.Fs, remote, content string) f
 }
 
 func (r *run) writeRemoteBytes(t *testing.T, f fs.Fs, remote string, data []byte) {
-	r.writeObjectBytes(t, f, remote, data)
+	var err error
+
+	if r.useMount {
+		err = r.retryBlock(func() error {
+			return ioutil.WriteFile(path.Join(r.mntDir, remote), data, 0600)
+		}, 3, time.Second*3)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else {
+		r.writeObjectBytes(t, f, remote, data)
+	}
 }
 
 func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.ReadCloser) {
-	r.writeObjectReader(t, f, remote, in)
+	defer func() {
+		_ = in.Close()
+	}()
+
+	if r.useMount {
+		out, err := os.Create(path.Join(r.mntDir, remote))
+		require.NoError(t, err)
+		defer func() {
+			_ = out.Close()
+		}()
+
+		_, err = io.Copy(out, in)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else {
+		r.writeObjectReader(t, f, remote, in)
+	}
 }
 
 func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {
 	in := bytes.NewReader(data)
 	_ = r.writeObjectReader(t, f, remote, in)
-	o, err := f.NewObject(context.Background(), remote)
+	o, err := f.NewObject(remote)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data)), o.Size())
 	return o
@@ -1020,8 +1218,12 @@ func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte
 func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object {
 	modTime := time.Now()
 	objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f)
-	obj, err := f.Put(context.Background(), in, objInfo)
+	obj, err := f.Put(in, objInfo)
 	require.NoError(t, err)
+	if r.useMount {
+		r.vfs.WaitForWriters(10 * time.Second)
+	}
+
 	return obj
 }
 
@@ -1029,16 +1231,26 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
 	var err error
 	var obj fs.Object
 
-	in1 := bytes.NewReader(data1)
-	in2 := bytes.NewReader(data2)
-	objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
-	objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
+	if r.useMount {
+		err = ioutil.WriteFile(path.Join(r.mntDir, remote), data1, 0600)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+		err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+		obj, err = f.NewObject(remote)
+	} else {
+		in1 := bytes.NewReader(data1)
+		in2 := bytes.NewReader(data2)
+		objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
+		objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
 
-	_, err = f.Put(context.Background(), in1, objInfo1)
-	require.NoError(t, err)
-	obj, err = f.NewObject(context.Background(), remote)
-	require.NoError(t, err)
-	err = obj.Update(context.Background(), in2, objInfo2)
+		obj, err = f.Put(in1, objInfo1)
+		require.NoError(t, err)
+		obj, err = f.NewObject(remote)
+		require.NoError(t, err)
+		err = obj.Update(in2, objInfo2)
+	}
 	require.NoError(t, err)
 
 	return obj
@@ -1048,12 +1260,30 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 	size := end - offset
 	checkSample := make([]byte, size)
 
-	co, err := f.NewObject(context.Background(), remote)
-	if err != nil {
-		return checkSample, err
+	if r.useMount {
+		f, err := os.Open(path.Join(r.mntDir, remote))
+		defer func() {
+			_ = f.Close()
+		}()
+		if err != nil {
+			return checkSample, err
+		}
+		_, _ = f.Seek(offset, io.SeekStart)
+		totalRead, err := io.ReadFull(f, checkSample)
+		checkSample = checkSample[:totalRead]
+		if err == io.EOF || err == io.ErrUnexpectedEOF {
+			err = nil
+		}
+		if err != nil {
+			return checkSample, err
+		}
+	} else {
+		co, err := f.NewObject(remote)
+		if err != nil {
+			return checkSample, err
+		}
+		checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck)
 	}
-	checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck)
-
 	if !noLengthCheck && size != int64(len(checkSample)) {
 		return checkSample, errors.Errorf("read size doesn't match expected: %v <> %v", len(checkSample), size)
 	}
@@ -1063,7 +1293,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte {
 	size := end - offset
 	checkSample := make([]byte, size)
-	reader, err := o.Open(context.Background(), &fs.SeekOption{Offset: offset})
+	reader, err := o.Open(&fs.SeekOption{Offset: offset})
 	require.NoError(t, err)
 	totalRead, err := io.ReadFull(reader, checkSample)
 	if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck {
@@ -1076,19 +1306,28 @@ func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLe
 }
 
 func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) {
-	err := f.Mkdir(context.Background(), remote)
+	var err error
+	if r.useMount {
+		err = os.Mkdir(path.Join(r.mntDir, remote), 0700)
+	} else {
+		err = f.Mkdir(remote)
+	}
 	require.NoError(t, err)
 }
 
 func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 	var err error
 
-	var obj fs.Object
-	obj, err = f.NewObject(context.Background(), remote)
-	if err != nil {
-		err = f.Rmdir(context.Background(), remote)
+	if r.useMount {
+		err = os.Remove(path.Join(r.mntDir, remote))
 	} else {
-		err = obj.Remove(context.Background())
+		var obj fs.Object
+		obj, err = f.NewObject(remote)
+		if err != nil {
+			err = f.Rmdir(remote)
+		} else {
+			err = obj.Remove()
+		}
 	}
 
 	return err
@@ -1097,14 +1336,42 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) {
 	var err error
 	var l []interface{}
-	var list fs.DirEntries
-	list, err = f.List(context.Background(), remote)
-	for _, ll := range list {
-		l = append(l, ll)
+	if r.useMount {
+		var list []os.FileInfo
+		list, err = ioutil.ReadDir(path.Join(r.mntDir, remote))
+		for _, ll := range list {
+			l = append(l, ll)
+		}
+	} else {
+		var list fs.DirEntries
+		list, err = f.List(remote)
+		for _, ll := range list {
+			l = append(l, ll)
+		}
 	}
 	return l, err
 }
 
+func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
+	var err error
+	var l []string
+	if r.useMount {
+		var list []os.FileInfo
+		list, err = ioutil.ReadDir(path.Join(r.mntDir, remote))
+		for _, ll := range list {
+			l = append(l, ll.Name())
+		}
+	} else {
+		var list fs.DirEntries
+		list, err = f.List(remote)
+		for _, ll := range list {
+			l = append(l, ll.Remote())
+		}
+	}
+	require.NoError(t, err)
+	return l
+}
+
 func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
 	in, err := os.Open(src)
 	if err != nil {
@@ -1129,8 +1396,14 @@ func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().DirMove != nil {
-		err = rootFs.Features().DirMove(context.Background(), rootFs, src, dst)
+	if runInstance.useMount {
+		err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst))
+		if err != nil {
+			return err
+		}
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().DirMove != nil {
+		err = rootFs.Features().DirMove(rootFs, src, dst)
 		if err != nil {
 			return err
 		}
@@ -1145,12 +1418,18 @@ func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().Move != nil {
-		obj1, err := rootFs.NewObject(context.Background(), src)
+	if runInstance.useMount {
+		err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst))
 		if err != nil {
 			return err
 		}
-		_, err = rootFs.Features().Move(context.Background(), obj1, dst)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().Move != nil {
+		obj1, err := rootFs.NewObject(src)
+		if err != nil {
+			return err
+		}
+		_, err = rootFs.Features().Move(obj1, dst)
 		if err != nil {
 			return err
 		}
@@ -1165,12 +1444,18 @@ func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().Copy != nil {
-		obj, err := rootFs.NewObject(context.Background(), src)
+	if r.useMount {
+		err = r.copyFile(t, rootFs, path.Join(r.mntDir, src), path.Join(r.mntDir, dst))
 		if err != nil {
 			return err
 		}
-		_, err = rootFs.Features().Copy(context.Background(), obj, dst)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().Copy != nil {
+		obj, err := rootFs.NewObject(src)
+		if err != nil {
+			return err
+		}
+		_, err = rootFs.Features().Copy(obj, dst)
 		if err != nil {
 			return err
 		}
@@ -1185,17 +1470,31 @@ func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error) {
 	var err error
 
-	obj1, err := rootFs.NewObject(context.Background(), src)
+	if r.useMount {
+		fi, err := os.Stat(path.Join(runInstance.mntDir, src))
+		if err != nil {
+			return time.Time{}, err
+		}
+		return fi.ModTime(), nil
+	}
+	obj1, err := rootFs.NewObject(src)
 	if err != nil {
 		return time.Time{}, err
 	}
-	return obj1.ModTime(context.Background()), nil
+	return obj1.ModTime(), nil
 }
 
 func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
 	var err error
 
-	obj1, err := rootFs.NewObject(context.Background(), src)
+	if r.useMount {
+		fi, err := os.Stat(path.Join(runInstance.mntDir, src))
+		if err != nil {
+			return int64(0), err
+		}
+		return fi.Size(), nil
+	}
+	obj1, err := rootFs.NewObject(src)
 	if err != nil {
 		return int64(0), err
 	}
@@ -1205,15 +1504,26 @@ func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
 func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) error {
 	var err error
 
-	var obj1 fs.Object
-	obj1, err = rootFs.NewObject(context.Background(), src)
-	if err != nil {
-		return err
+	if r.useMount {
+		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = f.Close()
+			r.vfs.WaitForWriters(10 * time.Second)
+		}()
+		_, err = f.WriteString(data + append)
+	} else {
+		obj1, err := rootFs.NewObject(src)
+		if err != nil {
+			return err
+		}
+		data1 := []byte(data + append)
+		r := bytes.NewReader(data1)
+		objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
+		err = obj1.Update(r, objInfo1)
 	}
-	data1 := []byte(data + append)
-	reader := bytes.NewReader(data1)
-	objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
-	err = obj1.Update(context.Background(), reader, objInfo1)
 
 	return err
 }
@@ -1337,13 +1647,15 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	}
-	if f.Features().UnWrap != nil {
-		cfs, ok := f.Features().UnWrap().(*cache.Fs)
-		if ok {
-			return cfs, nil
+	} else {
+		if f.Features().UnWrap != nil {
+			cfs, ok := f.Features().UnWrap().(*cache.Fs)
+			if ok {
+				return cfs, nil
+			}
 		}
 	}
+
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/cache_mount_unix_test.go

@@ -0,0 +1,78 @@
+// +build !plan9,!windows
+
+package cache_test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"bazil.org/fuse"
+	fusefs "bazil.org/fuse/fs"
+	"github.com/ncw/rclone/cmd/mount"
+	"github.com/ncw/rclone/cmd/mountlib"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	device := f.Name() + ":" + f.Root()
+	var options = []fuse.MountOption{
+		fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)),
+		fuse.Subtype("rclone"),
+		fuse.FSName(device), fuse.VolumeName(device),
+		fuse.NoAppleDouble(),
+		fuse.NoAppleXattr(),
+		//fuse.AllowOther(),
+	}
+	err := os.MkdirAll(r.mntDir, os.ModePerm)
+	require.NoError(t, err)
+	c, err := fuse.Mount(r.mntDir, options...)
+	require.NoError(t, err)
+	filesys := mount.NewFS(f)
+	server := fusefs.New(c, nil)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		err := server.Serve(filesys)
+		closeErr := c.Close()
+		if err == nil {
+			err = closeErr
+		}
+		r.unmountRes <- err
+	}()
+
+	// check if the mount process has an error to report
+	<-c.Ready
+	require.NoError(t, c.MountError)
+
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		filesys.VFS.Shutdown()
+		return fuse.Unmount(r.mntDir)
+	}
+
+	r.vfs = filesys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_mount_windows_test.go

@@ -0,0 +1,124 @@
+// +build windows
+
+package cache_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/billziss-gh/cgofuse/fuse"
+	"github.com/ncw/rclone/cmd/cmount"
+	"github.com/ncw/rclone/cmd/mountlib"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+)
+
+// waitFor runs fn() until it returns true or the timeout expires
+func waitFor(fn func() bool) (ok bool) {
+	const totalWait = 10 * time.Second
+	const individualWait = 10 * time.Millisecond
+	for i := 0; i < int(totalWait/individualWait); i++ {
+		ok = fn()
+		if ok {
+			return ok
+		}
+		time.Sleep(individualWait)
+	}
+	return false
+}
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+
+	device := f.Name() + ":" + f.Root()
+	options := []string{
+		"-o", "fsname=" + device,
+		"-o", "subtype=rclone",
+		"-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead),
+		"-o", "uid=-1",
+		"-o", "gid=-1",
+		"-o", "allow_other",
+		// This causes FUSE to supply O_TRUNC with the Open
+		// call which is more efficient for cmount.  However
+		// it does not work with cgofuse on Windows with
+		// WinFSP so cmount must work with or without it.
+		"-o", "atomic_o_trunc",
+		"--FileSystemName=rclone",
+	}
+
+	fsys := cmount.NewFS(f)
+	host := fuse.NewFileSystemHost(fsys)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		var err error
+		ok := host.Mount(r.mntDir, options)
+		if !ok {
+			err = errors.New("mount failed")
+		}
+		r.unmountRes <- err
+	}()
+
+	// unmount
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		fsys.VFS.Shutdown()
+		if host.Unmount() {
+			if !waitFor(func() bool {
+				_, err := os.Stat(r.mntDir)
+				return err != nil
+			}) {
+				t.Fatalf("mountpoint %q didn't disappear after unmount - continuing anyway", r.mntDir)
+			}
+			return nil
+		}
+		return errors.New("host unmount failed")
+	}
+
+	// Wait for the filesystem to become ready, checking the file
+	// system didn't blow up before starting
+	select {
+	case err := <-r.unmountRes:
+		require.NoError(t, err)
+	case <-time.After(time.Second * 3):
+	}
+
+	// Wait for the mount point to be available on Windows
+	// On Windows the Init signal comes slightly before the mount is ready
+	if !waitFor(func() bool {
+		_, err := os.Stat(r.mntDir)
+		return err == nil
+	}) {
+		t.Errorf("mountpoint %q didn't became available on mount", r.mntDir)
+	}
+
+	r.vfs = fsys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_test.go

@@ -1,25 +1,21 @@
 // Test Cache filesystem interface
 
-// +build !plan9,!js
-// +build !race
+// +build !plan9
 
 package cache_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   "TestCache:",
-		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
-		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
+		RemoteName: "TestCache:",
+		NilObject:  (*cache.Object)(nil),
 	})
 }

backend/cache/cache_unsupported.go

@@ -1,6 +1,6 @@
 // Build for cache for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 js
+// +build plan9
 
 package cache

backend/cache/cache_upload_test.go

@@ -1,11 +1,8 @@
-// +build !plan9,!js
-// +build !race
+// +build !plan9
 
 package cache_test
 
 import (
-	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"path"
@@ -13,9 +10,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/drive"
-	"github.com/rclone/rclone/fs"
+	"fmt"
+
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -87,11 +86,11 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "one")
+	err := rootFs.Mkdir("one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "one/test")
+	err = rootFs.Mkdir("one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "second")
+	err = rootFs.Mkdir("second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -124,11 +123,11 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "one")
+	err := rootFs.Mkdir("one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "one/test")
+	err = rootFs.Mkdir("one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "second")
+	err = rootFs.Mkdir("second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -167,7 +166,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "test")
+	err := rootFs.Mkdir("test")
 	require.NoError(t, err)
 	minSize := 5242880
 	maxSize := 10485760
@@ -235,9 +234,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "second/one")
+		_, err = rootFs.NewObject("second/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -258,7 +257,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "directory not empty")
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it exists in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -272,9 +271,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.NoError(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/second")
+		_, err = rootFs.NewObject("test/second")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -291,9 +290,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/third")
+		_, err = rootFs.NewObject("test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -308,7 +307,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	// test Remove -- allowed
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.NoError(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.Error(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -320,7 +319,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	require.NoError(t, err)
 	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
 	require.NoError(t, err)
-	obj2, err := rootFs.NewObject(context.Background(), "test/one")
+	obj2, err := rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
 	require.Equal(t, "one content updated", string(data2))
@@ -368,7 +367,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -380,7 +379,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Rmdir
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -391,9 +390,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.Error(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/second")
+		_, err = rootFs.NewObject("test/second")
 		require.Error(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -406,9 +405,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/third")
+		_, err = rootFs.NewObject("test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -423,7 +422,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Remove
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.Error(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))

backend/cache/cache_upload_test.go.orig

@@ -0,0 +1,455 @@
+// +build !plan9
+
+package cache_test
+
+import (
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"fmt"
+
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalUploadTempDirCreated(t *testing.T) {
+	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
+	require.NoError(t, err)
+}
+
+func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
+	// create some rand test data
+	testSize := int64(524288000)
+	testReader := runInstance.randomReader(t, testSize)
+	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
+	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
+	// validate that it exists in temp fs
+	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.NoError(t, err)
+
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(524416032), ti.Size())
+	} else {
+		require.Equal(t, testSize, ti.Size())
+	}
+	de1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	runInstance.completeBackgroundUpload(t, "one", bu)
+	// check if it was removed from temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if it can be read
+	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
+	require.NoError(t, err)
+	require.Len(t, data2, 1024)
+}
+
+func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadMoveExistingFile(t *testing.T) {
+	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(10485760)
+	testReader := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	time.Sleep(time.Second * 5)
+	//_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	//require.NoError(t, err)
+
+	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
+	require.NoError(t, err)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadTempPathCleaned(t *testing.T) {
+	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(1048576)
+	testReader := runInstance.randomReader(t, testSize)
+	testReader2 := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2)
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second")))
+	require.False(t, os.IsNotExist(err))
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin")))
+	require.True(t, os.IsNotExist(err))
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadQueueMoreFiles(t *testing.T) {
+	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("test")
+	require.NoError(t, err)
+	minSize := 5242880
+	maxSize := 10485760
+	totalFiles := 10
+	rand.Seed(time.Now().Unix())
+
+	lastFile := ""
+	for i := 0; i < totalFiles; i++ {
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		testReader := runInstance.randomReader(t, size)
+		remote := "test/" + strconv.Itoa(i) + ".bin"
+		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
+
+		// validate that it exists in temp fs
+		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
+		require.NoError(t, err)
+		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
+
+		if runInstance.wrappedIsExternal && i < totalFiles-1 {
+			time.Sleep(time.Second * 3)
+		}
+		lastFile = remote
+	}
+
+	// check if cache lists all files, likely temp upload didn't finish yet
+	de1, err := runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+
+	// wait for background uploader to do its thing
+	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
+
+	// retry until we have no more temp files and fail if they don't go down to 0
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if cache lists all files
+	de1, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+}
+
+func TestInternalUploadTempFileOperations(t *testing.T) {
+	id := "tiutfo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove - allowed
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("second/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.NoError(t, err)
+		_, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+		require.Error(t, err)
+		var started bool
+		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
+		require.NoError(t, err)
+		require.False(t, started)
+		runInstance.mkdir(t, rootFs, "test")
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Rmdir - allowed
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "directory not empty")
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+	require.False(t, started)
+	require.NoError(t, err)
+
+	// test Move/Rename -- allowed
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.NoError(t, err)
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove -- allowed
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.NoError(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.Error(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.Error(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update -- allowed
+	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
+	require.NoError(t, err)
+	obj2, err := rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
+	require.Equal(t, "one content updated", string(data2))
+	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(67), tmpInfo.Size())
+	} else {
+		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	}
+
+	// test SetModTime -- allowed
+	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	require.NotEqual(t, secondModTime, firstModTime)
+	require.NotEqual(t, time.Time{}, firstModTime)
+	require.NotEqual(t, time.Time{}, secondModTime)
+}
+
+func TestInternalUploadUploadingFileOperations(t *testing.T) {
+	id := "tiuufo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.Error(t, err)
+	}
+
+	// test Rmdir
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test Move/Rename
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.Error(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.Error(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.Error(t, err)
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update - this seems to work. Why? FIXME
+	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
+	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
+	//	require.Equal(t, "one content", string(data2))
+	//
+	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	//	require.NoError(t, err)
+	//	if runInstance.rootIsCrypt {
+	//		require.Equal(t, int64(67), tmpInfo.Size())
+	//	} else {
+	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	//	}
+	//})
+	//require.Error(t, err)
+
+	// test SetModTime -- seems to work cause of previous
+	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//require.Equal(t, secondModTime, firstModTime)
+	//require.NotEqual(t, time.Time{}, firstModTime)
+	//require.NotEqual(t, time.Time{}, secondModTime)
+}

backend/cache/cache_upload_test.go.rej

@@ -0,0 +1,12 @@
+--- cache_upload_test.go
++++ cache_upload_test.go
+@@ -1500,9 +1469,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
+ 	}
+ 	r.tempFiles = nil
+ 	debug.FreeOSMemory()
+-	for k, v := range r.runDefaultFlagMap {
+-		_ = flag.Set(k, v)
+-	}
+ }
+ 
+ func (r *run) randomBytes(t *testing.T, size int64) []byte {

backend/cache/directory.go

@@ -1,13 +1,13 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
-	"path"
 	"time"
 
-	"github.com/rclone/rclone/fs"
+	"path"
+
+	"github.com/ncw/rclone/fs"
 )
 
 // Directory is a generic dir that stores basic information about it
@@ -56,7 +56,7 @@ func ShallowDirectory(f *Fs, remote string) *Directory {
 }
 
 // DirectoryFromOriginal builds one from a generic fs.Directory
-func DirectoryFromOriginal(ctx context.Context, f *Fs, d fs.Directory) *Directory {
+func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory {
 	var cd *Directory
 	fullRemote := path.Join(f.Root(), d.Remote())
 
@@ -68,7 +68,7 @@ func DirectoryFromOriginal(ctx context.Context, f *Fs, d fs.Directory) *Director
 		CacheFs:      f,
 		Name:         name,
 		Dir:          dir,
-		CacheModTime: d.ModTime(ctx).UnixNano(),
+		CacheModTime: d.ModTime().UnixNano(),
 		CacheSize:    d.Size(),
 		CacheItems:   d.Items(),
 		CacheType:    "Directory",
@@ -101,8 +101,17 @@ func (d *Directory) abs() string {
 	return cleanPath(path.Join(d.Dir, d.Name))
 }
 
+// parentRemote returns the absolute path parent remote
+func (d *Directory) parentRemote() string {
+	absPath := d.abs()
+	if absPath == "" {
+		return ""
+	}
+	return cleanPath(path.Dir(absPath))
+}
+
 // ModTime returns the cached ModTime
-func (d *Directory) ModTime(ctx context.Context) time.Time {
+func (d *Directory) ModTime() time.Time {
 	return time.Unix(0, d.CacheModTime)
 }
 

backend/cache/handle.go

@@ -1,20 +1,20 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
 	"fmt"
 	"io"
-	"path"
-	"runtime"
-	"strings"
 	"sync"
 	"time"
 
+	"path"
+	"runtime"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/operations"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/operations"
 )
 
 var uploaderMap = make(map[string]*backgroundWriter)
@@ -41,7 +41,6 @@ func initBackgroundUploader(fs *Fs) (*backgroundWriter, error) {
 
 // Handle is managing the read/write/seek operations on an open handle
 type Handle struct {
-	ctx            context.Context
 	cachedObject   *Object
 	cfs            *Fs
 	memory         *Memory
@@ -50,19 +49,17 @@ type Handle struct {
 	offset         int64
 	seenOffsets    map[int64]bool
 	mu             sync.Mutex
-	workersWg      sync.WaitGroup
 	confirmReading chan bool
-	workers        int
-	maxWorkerID    int
-	UseMemory      bool
-	closed         bool
-	reading        bool
+
+	UseMemory bool
+	workers   []*worker
+	closed    bool
+	reading   bool
 }
 
 // NewObjectHandle returns a new Handle for an existing Object
-func NewObjectHandle(ctx context.Context, o *Object, cfs *Fs) *Handle {
+func NewObjectHandle(o *Object, cfs *Fs) *Handle {
 	r := &Handle{
-		ctx:           ctx,
 		cachedObject:  o,
 		cfs:           cfs,
 		offset:        0,
@@ -98,7 +95,7 @@ func (r *Handle) String() string {
 
 // startReadWorkers will start the worker pool
 func (r *Handle) startReadWorkers() {
-	if r.workers > 0 {
+	if r.hasAtLeastOneWorker() {
 		return
 	}
 	totalWorkers := r.cacheFs().opt.TotalWorkers
@@ -120,27 +117,26 @@ func (r *Handle) startReadWorkers() {
 
 // scaleOutWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
-	current := r.workers
+	current := len(r.workers)
 	if current == desired {
 		return
 	}
 	if current > desired {
 		// scale in gracefully
-		for r.workers > desired {
+		for i := 0; i < current-desired; i++ {
 			r.preloadQueue <- -1
-			r.workers--
 		}
 	} else {
 		// scale out
-		for r.workers < desired {
+		for i := 0; i < desired-current; i++ {
 			w := &worker{
 				r:  r,
-				id: r.maxWorkerID,
+				ch: r.preloadQueue,
+				id: current + i,
 			}
-			r.workersWg.Add(1)
-			r.workers++
-			r.maxWorkerID++
 			go w.run()
+
+			r.workers = append(r.workers, w)
 		}
 	}
 	// ignore first scale out from 0
@@ -152,7 +148,7 @@ func (r *Handle) scaleWorkers(desired int) {
 func (r *Handle) confirmExternalReading() {
 	// if we have a max value of workers
 	// then we skip this step
-	if r.workers > 1 ||
+	if len(r.workers) > 1 ||
 		!r.cacheFs().plexConnector.isConfigured() {
 		return
 	}
@@ -182,7 +178,7 @@ func (r *Handle) queueOffset(offset int64) {
 			}
 		}
 
-		for i := 0; i < r.workers; i++ {
+		for i := 0; i < len(r.workers); i++ {
 			o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
 			if o < 0 || o >= r.cachedObject.Size() {
 				continue
@@ -197,6 +193,16 @@ func (r *Handle) queueOffset(offset int64) {
 	}
 }
 
+func (r *Handle) hasAtLeastOneWorker() bool {
+	oneWorker := false
+	for i := 0; i < len(r.workers); i++ {
+		if r.workers[i].isRunning() {
+			oneWorker = true
+		}
+	}
+	return oneWorker
+}
+
 // getChunk is called by the FS to retrieve a specific chunk of known start and size from where it can find it
 // it can be from transient or persistent cache
 // it will also build the chunk from the cache's specific chunk boundaries and build the final desired chunk in a buffer
@@ -237,7 +243,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	// not found in ram or
 	// the worker didn't managed to download the chunk in time so we abort and close the stream
 	if err != nil || len(data) == 0 || !found {
-		if r.workers == 0 {
+		if !r.hasAtLeastOneWorker() {
 			fs.Errorf(r, "out of workers")
 			return nil, io.ErrUnexpectedEOF
 		}
@@ -298,7 +304,14 @@ func (r *Handle) Close() error {
 	close(r.preloadQueue)
 	r.closed = true
 	// wait for workers to complete their jobs before returning
-	r.workersWg.Wait()
+	waitCount := 3
+	for i := 0; i < len(r.workers); i++ {
+		waitIdx := 0
+		for r.workers[i].isRunning() && waitIdx < waitCount {
+			time.Sleep(time.Second)
+			waitIdx++
+		}
+	}
 	r.memory.db.Flush()
 
 	fs.Debugf(r, "cache reader closed %v", r.offset)
@@ -335,9 +348,12 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 }
 
 type worker struct {
-	r  *Handle
-	rc io.ReadCloser
-	id int
+	r       *Handle
+	ch      <-chan int64
+	rc      io.ReadCloser
+	id      int
+	running bool
+	mu      sync.Mutex
 }
 
 // String is a representation of this worker
@@ -354,7 +370,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 	r := w.rc
 	if w.rc == nil {
 		r, err = w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-			return w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
+			return w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
 		})
 		if err != nil {
 			return nil, err
@@ -364,7 +380,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	if !closeOpen {
 		if do, ok := r.(fs.RangeSeeker); ok {
-			_, err = do.RangeSeek(w.r.ctx, offset, io.SeekStart, end-offset)
+			_, err = do.RangeSeek(offset, io.SeekStart, end-offset)
 			return r, err
 		} else if do, ok := r.(io.Seeker); ok {
 			_, err = do.Seek(offset, io.SeekStart)
@@ -374,7 +390,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	_ = w.rc.Close()
 	return w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-		r, err = w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
+		r, err = w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
 		if err != nil {
 			return nil, err
 		}
@@ -382,19 +398,33 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 	})
 }
 
+func (w *worker) isRunning() bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.running
+}
+
+func (w *worker) setRunning(f bool) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.running = f
+}
+
 // run is the main loop for the worker which receives offsets to preload
 func (w *worker) run() {
 	var err error
 	var data []byte
+	defer w.setRunning(false)
 	defer func() {
 		if w.rc != nil {
 			_ = w.rc.Close()
+			w.setRunning(false)
 		}
-		w.r.workersWg.Done()
 	}()
 
 	for {
-		chunkStart, open := <-w.r.preloadQueue
+		chunkStart, open := <-w.ch
+		w.setRunning(true)
 		if chunkStart < 0 || !open {
 			break
 		}
@@ -452,7 +482,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	// we seem to be getting only errors so we abort
 	if err != nil {
 		fs.Errorf(w, "object open failed %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
+		err = w.r.cachedObject.refreshFromSource(true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -465,7 +495,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	sourceRead, err = io.ReadFull(w.rc, data)
 	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
+		err = w.r.cachedObject.refreshFromSource(true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -591,7 +621,7 @@ func (b *backgroundWriter) run() {
 		remote := b.fs.cleanRootFromPath(absPath)
 		b.notify(remote, BackgroundUploadStarted, nil)
 		fs.Infof(remote, "background upload: started upload")
-		err = operations.MoveFile(context.TODO(), b.fs.UnWrap(), b.fs.tempFs, remote, remote)
+		err = operations.MoveFile(b.fs.UnWrap(), b.fs.tempFs, remote, remote)
 		if err != nil {
 			b.notify(remote, BackgroundUploadError, err)
 			_ = b.fs.cache.rollbackPendingUpload(absPath)
@@ -601,14 +631,14 @@ func (b *backgroundWriter) run() {
 		// clean empty dirs up to root
 		thisDir := cleanPath(path.Dir(remote))
 		for thisDir != "" {
-			thisList, err := b.fs.tempFs.List(context.TODO(), thisDir)
+			thisList, err := b.fs.tempFs.List(thisDir)
 			if err != nil {
 				break
 			}
 			if len(thisList) > 0 {
 				break
 			}
-			err = b.fs.tempFs.Rmdir(context.TODO(), thisDir)
+			err = b.fs.tempFs.Rmdir(thisDir)
 			fs.Debugf(thisDir, "cleaned from temp path")
 			if err != nil {
 				break

backend/cache/object.go

@@ -1,18 +1,17 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
 	"io"
 	"path"
 	"sync"
 	"time"
 
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/readers"
 )
 
 const (
@@ -24,16 +23,15 @@ const (
 type Object struct {
 	fs.Object `json:"-"`
 
-	ParentFs      fs.Fs     `json:"-"`        // parent fs
-	CacheFs       *Fs       `json:"-"`        // cache fs
-	Name          string    `json:"name"`     // name of the directory
-	Dir           string    `json:"dir"`      // abs path of the object
-	CacheModTime  int64     `json:"modTime"`  // modification or creation time - IsZero for unknown
-	CacheSize     int64     `json:"size"`     // size of directory and contents or -1 if unknown
-	CacheStorable bool      `json:"storable"` // says whether this object can be stored
-	CacheType     string    `json:"cacheType"`
-	CacheTs       time.Time `json:"cacheTs"`
-	cacheHashesMu sync.Mutex
+	ParentFs      fs.Fs                `json:"-"`        // parent fs
+	CacheFs       *Fs                  `json:"-"`        // cache fs
+	Name          string               `json:"name"`     // name of the directory
+	Dir           string               `json:"dir"`      // abs path of the object
+	CacheModTime  int64                `json:"modTime"`  // modification or creation time - IsZero for unknown
+	CacheSize     int64                `json:"size"`     // size of directory and contents or -1 if unknown
+	CacheStorable bool                 `json:"storable"` // says whether this object can be stored
+	CacheType     string               `json:"cacheType"`
+	CacheTs       time.Time            `json:"cacheTs"`
 	CacheHashes   map[hash.Type]string // all supported hashes cached
 
 	refreshMutex sync.Mutex
@@ -70,7 +68,7 @@ func NewObject(f *Fs, remote string) *Object {
 }
 
 // ObjectFromOriginal builds one from a generic fs.Object
-func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
+func ObjectFromOriginal(f *Fs, o fs.Object) *Object {
 	var co *Object
 	fullRemote := cleanPath(path.Join(f.Root(), o.Remote()))
 	dir, name := path.Split(fullRemote)
@@ -94,19 +92,17 @@ func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
 		CacheType: cacheType,
 		CacheTs:   time.Now(),
 	}
-	co.updateData(ctx, o)
+	co.updateData(o)
 	return co
 }
 
-func (o *Object) updateData(ctx context.Context, source fs.Object) {
+func (o *Object) updateData(source fs.Object) {
 	o.Object = source
-	o.CacheModTime = source.ModTime(ctx).UnixNano()
+	o.CacheModTime = source.ModTime().UnixNano()
 	o.CacheSize = source.Size()
 	o.CacheStorable = source.Storable()
 	o.CacheTs = time.Now()
-	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
-	o.cacheHashesMu.Unlock()
 }
 
 // Fs returns its FS info
@@ -134,20 +130,20 @@ func (o *Object) abs() string {
 }
 
 // ModTime returns the cached ModTime
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	_ = o.refresh(ctx)
+func (o *Object) ModTime() time.Time {
+	_ = o.refresh()
 	return time.Unix(0, o.CacheModTime)
 }
 
 // Size returns the cached Size
 func (o *Object) Size() int64 {
-	_ = o.refresh(context.TODO())
+	_ = o.refresh()
 	return o.CacheSize
 }
 
 // Storable returns the cached Storable
 func (o *Object) Storable() bool {
-	_ = o.refresh(context.TODO())
+	_ = o.refresh()
 	return o.CacheStorable
 }
 
@@ -155,18 +151,18 @@ func (o *Object) Storable() bool {
 // all these conditions must be true to ignore a refresh
 // 1. cache ts didn't expire yet
 // 2. is not pending a notification from the wrapped fs
-func (o *Object) refresh(ctx context.Context) error {
+func (o *Object) refresh() error {
 	isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
 	isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
 	if !isExpired && !isNotified {
 		return nil
 	}
 
-	return o.refreshFromSource(ctx, true)
+	return o.refreshFromSource(true)
 }
 
 // refreshFromSource requests the original FS for the object in case it comes from a cached entry
-func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
+func (o *Object) refreshFromSource(force bool) error {
 	o.refreshMutex.Lock()
 	defer o.refreshMutex.Unlock()
 	var err error
@@ -176,29 +172,29 @@ func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
 		return nil
 	}
 	if o.isTempFile() {
-		liveObject, err = o.ParentFs.NewObject(ctx, o.Remote())
+		liveObject, err = o.ParentFs.NewObject(o.Remote())
 		err = errors.Wrapf(err, "in parent fs %v", o.ParentFs)
 	} else {
-		liveObject, err = o.CacheFs.Fs.NewObject(ctx, o.Remote())
+		liveObject, err = o.CacheFs.Fs.NewObject(o.Remote())
 		err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs)
 	}
 	if err != nil {
 		fs.Errorf(o, "error refreshing object in : %v", err)
 		return err
 	}
-	o.updateData(ctx, liveObject)
+	o.updateData(liveObject)
 	o.persist()
 
 	return nil
 }
 
 // SetModTime sets the ModTime of this object
-func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) SetModTime(t time.Time) error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 
-	err := o.Object.SetModTime(ctx, t)
+	err := o.Object.SetModTime(t)
 	if err != nil {
 		return err
 	}
@@ -211,19 +207,13 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
 }
 
 // Open is used to request a specific part of the file using fs.RangeOption
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	var err error
-
-	if o.Object == nil {
-		err = o.refreshFromSource(ctx, true)
-	} else {
-		err = o.refresh(ctx)
-	}
-	if err != nil {
+func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+	if err := o.refreshFromSource(true); err != nil {
 		return nil, err
 	}
 
-	cacheReader := NewObjectHandle(ctx, o, o.CacheFs)
+	var err error
+	cacheReader := NewObjectHandle(o, o.CacheFs)
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
 		switch x := option.(type) {
@@ -242,8 +232,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 }
 
 // Update will change the object data
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -258,7 +248,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	fs.Debugf(o, "updating object contents with size %v", src.Size())
 
 	// FIXME use reliable upload
-	err := o.Object.Update(ctx, in, src, options...)
+	err := o.Object.Update(in, src, options...)
 	if err != nil {
 		fs.Errorf(o, "error updating source: %v", err)
 		return err
@@ -269,11 +259,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// advertise to ChangeNotify if wrapped doesn't do that
 	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)
 
-	o.CacheModTime = src.ModTime(ctx).UnixNano()
+	o.CacheModTime = src.ModTime().UnixNano()
 	o.CacheSize = src.Size()
-	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
-	o.cacheHashesMu.Unlock()
 	o.CacheTs = time.Now()
 	o.persist()
 
@@ -281,8 +269,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove deletes the object from both the cache and the source
-func (o *Object) Remove(ctx context.Context) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) Remove() error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -294,7 +282,7 @@ func (o *Object) Remove(ctx context.Context) error {
 			return errors.Errorf("%v is currently uploading, can't delete", o)
 		}
 	}
-	err := o.Object.Remove(ctx)
+	err := o.Object.Remove()
 	if err != nil {
 		return err
 	}
@@ -312,27 +300,24 @@ func (o *Object) Remove(ctx context.Context) error {
 
 // Hash requests a hash of the object and stores in the cache
 // since it might or might not be called, this is lazy loaded
-func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
-	_ = o.refresh(ctx)
-	o.cacheHashesMu.Lock()
+func (o *Object) Hash(ht hash.Type) (string, error) {
+	_ = o.refresh()
 	if o.CacheHashes == nil {
 		o.CacheHashes = make(map[hash.Type]string)
 	}
+
 	cachedHash, found := o.CacheHashes[ht]
-	o.cacheHashesMu.Unlock()
 	if found {
 		return cachedHash, nil
 	}
-	if err := o.refreshFromSource(ctx, false); err != nil {
+	if err := o.refreshFromSource(false); err != nil {
 		return "", err
 	}
-	liveHash, err := o.Object.Hash(ctx, ht)
+	liveHash, err := o.Object.Hash(ht)
 	if err != nil {
 		return "", err
 	}
-	o.cacheHashesMu.Lock()
 	o.CacheHashes[ht] = liveHash
-	o.cacheHashesMu.Unlock()
 
 	o.persist()
 	fs.Debugf(o, "object hash cached: %v", liveHash)

backend/cache/plex.go

@@ -1,21 +1,22 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
 import (
-	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
-	cache "github.com/patrickmn/go-cache"
-	"github.com/rclone/rclone/fs"
+	"sync"
+
+	"bytes"
+	"io/ioutil"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/patrickmn/go-cache"
 	"golang.org/x/net/websocket"
 )
 
@@ -53,7 +54,6 @@ type plexConnector struct {
 	username   string
 	password   string
 	token      string
-	insecure   bool
 	f          *Fs
 	mu         sync.Mutex
 	running    bool
@@ -63,7 +63,7 @@ type plexConnector struct {
 }
 
 // newPlexConnector connects to a Plex server and generates a token
-func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool, saveToken func(string)) (*plexConnector, error) {
+func newPlexConnector(f *Fs, plexURL, username, password string, saveToken func(string)) (*plexConnector, error) {
 	u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/"))
 	if err != nil {
 		return nil, err
@@ -75,7 +75,6 @@ func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool,
 		username:   username,
 		password:   password,
 		token:      "",
-		insecure:   insecure,
 		stateCache: cache.New(time.Hour, time.Minute),
 		saveToken:  saveToken,
 	}
@@ -84,7 +83,7 @@ func newPlexConnector(f *Fs, plexURL, username, password string, insecure bool,
 }
 
 // newPlexConnector connects to a Plex server and generates a token
-func newPlexConnectorWithToken(f *Fs, plexURL, token string, insecure bool) (*plexConnector, error) {
+func newPlexConnectorWithToken(f *Fs, plexURL, token string) (*plexConnector, error) {
 	u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/"))
 	if err != nil {
 		return nil, err
@@ -94,7 +93,6 @@ func newPlexConnectorWithToken(f *Fs, plexURL, token string, insecure bool) (*pl
 		f:          f,
 		url:        u,
 		token:      token,
-		insecure:   insecure,
 		stateCache: cache.New(time.Hour, time.Minute),
 	}
 	pc.listenWebsocket()
@@ -109,26 +107,14 @@ func (p *plexConnector) closeWebsocket() {
 	p.running = false
 }
 
-func (p *plexConnector) websocketDial() (*websocket.Conn, error) {
-	u := strings.TrimRight(strings.Replace(strings.Replace(
-		p.url.String(), "http://", "ws://", 1), "https://", "wss://", 1), "/")
-	url := fmt.Sprintf(defPlexNotificationURL, u, p.token)
-
-	config, err := websocket.NewConfig(url, "http://localhost")
-	if err != nil {
-		return nil, err
-	}
-	if p.insecure {
-		config.TlsConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-	return websocket.DialConfig(config)
-}
-
 func (p *plexConnector) listenWebsocket() {
 	p.runningMu.Lock()
 	defer p.runningMu.Unlock()
 
-	conn, err := p.websocketDial()
+	u := strings.Replace(p.url.String(), "http://", "ws://", 1)
+	u = strings.Replace(u, "https://", "wss://", 1)
+	conn, err := websocket.Dial(fmt.Sprintf(defPlexNotificationURL, strings.TrimRight(u, "/"), p.token),
+		"", "http://localhost")
 	if err != nil {
 		fs.Errorf("plex", "%v", err)
 		return

backend/cache/storage_memory.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	cache "github.com/patrickmn/go-cache"
+	"github.com/ncw/rclone/fs"
+	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
 )
 
 // Memory is a wrapper of transient storage for a go-cache store

backend/cache/storage_persistent.go

@@ -1,25 +1,27 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
 import (
+	"time"
+
 	"bytes"
-	"context"
 	"encoding/binary"
 	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
+	"io/ioutil"
+
+	"fmt"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/walk"
-	bolt "go.etcd.io/bbolt"
 )
 
 // Constants
@@ -399,7 +401,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), encoded)
+		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -767,6 +769,31 @@ func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string)
 	return err
 }
 
+func (b *Persistent) dumpRoot() string {
+	var itBuckets func(buk *bolt.Bucket) map[string]interface{}
+
+	itBuckets = func(buk *bolt.Bucket) map[string]interface{} {
+		m := make(map[string]interface{})
+		c := buk.Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			if v == nil {
+				buk2 := buk.Bucket(k)
+				m[string(k)] = itBuckets(buk2)
+			} else {
+				m[string(k)] = "-"
+			}
+		}
+		return m
+	}
+	var mm map[string]interface{}
+	_ = b.db.View(func(tx *bolt.Tx) error {
+		mm = itBuckets(tx.Bucket([]byte(RootBucket)))
+		return nil
+	})
+	raw, _ := json.MarshalIndent(mm, "", "  ")
+	return string(raw)
+}
+
 // addPendingUpload adds a new file to the pending queue of uploads
 func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
@@ -785,7 +812,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), encoded)
+		err = bucket.Put([]byte(destPath), []byte(encoded))
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -980,8 +1007,17 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 	})
 }
 
+// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
+// TO BE USED IN TESTING ONLY
+func (b *Persistent) SetPendingUploadToStarted(remote string) error {
+	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
+		item.Started = true
+		return nil
+	})
+}
+
 // ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue
-func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) error {
+func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		_ = tx.DeleteBucket([]byte(tempBucket))
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
@@ -990,7 +1026,7 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(ctx, cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1016,7 +1052,7 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), encoded)
+			err = bucket.Put([]byte(destPath), []byte(encoded))
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}
@@ -1027,6 +1063,19 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 	})
 }
 
+// PurgeTempUploads will remove all the pending uploads from the queue
+// TO BE USED IN TESTING ONLY
+func (b *Persistent) PurgeTempUploads() {
+	b.tempQueueMux.Lock()
+	defer b.tempQueueMux.Unlock()
+
+	_ = b.db.Update(func(tx *bolt.Tx) error {
+		_ = tx.DeleteBucket([]byte(tempBucket))
+		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
+		return nil
+	})
+}
+
 // Close should be called when the program ends gracefully
 func (b *Persistent) Close() {
 	b.cleanupMux.Lock()

backend/cache/utils_test.go

@@ -1,23 +0,0 @@
-package cache
-
-import bolt "go.etcd.io/bbolt"
-
-// PurgeTempUploads will remove all the pending uploads from the queue
-func (b *Persistent) PurgeTempUploads() {
-	b.tempQueueMux.Lock()
-	defer b.tempQueueMux.Unlock()
-
-	_ = b.db.Update(func(tx *bolt.Tx) error {
-		_ = tx.DeleteBucket([]byte(tempBucket))
-		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
-		return nil
-	})
-}
-
-// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
-func (b *Persistent) SetPendingUploadToStarted(remote string) error {
-	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
-		item.Started = true
-		return nil
-	})
-}

backend/chunker/chunker_internal_test.go

@@ -1,881 +0,0 @@
-package chunker
-
-import (
-	"bytes"
-	"context"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"path"
-	"regexp"
-	"strings"
-	"testing"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Command line flags
-var (
-	UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
-)
-
-// test that chunking does not break large uploads
-func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
-	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
-		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
-			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
-			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
-			Size:    int64(kilobytes) * int64(fs.KibiByte),
-		})
-	})
-}
-
-// test chunk name parser
-func testChunkNameFormat(t *testing.T, f *Fs) {
-	saveOpt := f.opt
-	defer func() {
-		// restore original settings (f is pointer, f.opt is struct)
-		f.opt = saveOpt
-		_ = f.setChunkNameFormat(f.opt.NameFormat)
-	}()
-
-	assertFormat := func(pattern, wantDataFormat, wantCtrlFormat, wantNameRegexp string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.NoError(t, err)
-		assert.Equal(t, wantDataFormat, f.dataNameFmt)
-		assert.Equal(t, wantCtrlFormat, f.ctrlNameFmt)
-		assert.Equal(t, wantNameRegexp, f.nameRegexp.String())
-	}
-
-	assertFormatValid := func(pattern string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.NoError(t, err)
-	}
-
-	assertFormatInvalid := func(pattern string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.Error(t, err)
-	}
-
-	assertMakeName := func(wantChunkName, mainName string, chunkNo int, ctrlType, xactID string) {
-		gotChunkName := ""
-		assert.NotPanics(t, func() {
-			gotChunkName = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
-		}, "makeChunkName(%q,%d,%q,%q) must not panic", mainName, chunkNo, ctrlType, xactID)
-		if gotChunkName != "" {
-			assert.Equal(t, wantChunkName, gotChunkName)
-		}
-	}
-
-	assertMakeNamePanics := func(mainName string, chunkNo int, ctrlType, xactID string) {
-		assert.Panics(t, func() {
-			_ = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
-		}, "makeChunkName(%q,%d,%q,%q) should panic", mainName, chunkNo, ctrlType, xactID)
-	}
-
-	assertParseName := func(fileName, wantMainName string, wantChunkNo int, wantCtrlType, wantXactID string) {
-		gotMainName, gotChunkNo, gotCtrlType, gotXactID := f.parseChunkName(fileName)
-		assert.Equal(t, wantMainName, gotMainName)
-		assert.Equal(t, wantChunkNo, gotChunkNo)
-		assert.Equal(t, wantCtrlType, gotCtrlType)
-		assert.Equal(t, wantXactID, gotXactID)
-	}
-
-	const newFormatSupported = false // support for patterns not starting with base name (*)
-
-	// valid formats
-	assertFormat(`*.rclone_chunk.###`, `%s.rclone_chunk.%03d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*.rclone_chunk.#`, `%s.rclone_chunk.%d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*_chunk_#####`, `%s_chunk_%05d`, `%s_chunk__%s`, `^(.+?)_chunk_(?:([0-9]{5,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*-chunk-#`, `%s-chunk-%d`, `%s-chunk-_%s`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*-chunk-#-%^$()[]{}.+-!?:\`, `%s-chunk-%d-%%^$()[]{}.+-!?:\`, `%s-chunk-_%s-%%^$()[]{}.+-!?:\`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))-%\^\$\(\)\[\]\{\}\.\+-!\?:\\(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	if newFormatSupported {
-		assertFormat(`_*-chunk-##,`, `_%s-chunk-%02d,`, `_%s-chunk-_%s,`, `^_(.+?)-chunk-(?:([0-9]{2,})|_([a-z][a-z0-9]{2,6})),(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	}
-
-	// invalid formats
-	assertFormatInvalid(`chunk-#`)
-	assertFormatInvalid(`*-chunk`)
-	assertFormatInvalid(`*-*-chunk-#`)
-	assertFormatInvalid(`*-chunk-#-#`)
-	assertFormatInvalid(`#-chunk-*`)
-	assertFormatInvalid(`*/#`)
-
-	assertFormatValid(`*#`)
-	assertFormatInvalid(`**#`)
-	assertFormatInvalid(`#*`)
-	assertFormatInvalid(``)
-	assertFormatInvalid(`-`)
-
-	// quick tests
-	if newFormatSupported {
-		assertFormat(`part_*_#`, `part_%s_%d`, `part_%s__%s`, `^part_(.+?)_(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9][0-9a-z]{3,8})\.\.tmp_([0-9]{10,13}))?$`)
-		f.opt.StartFrom = 1
-
-		assertMakeName(`part_fish_1`, "fish", 0, "", "")
-		assertParseName(`part_fish_43`, "fish", 42, "", "")
-		assertMakeName(`part_fish__locks`, "fish", -2, "locks", "")
-		assertParseName(`part_fish__locks`, "fish", -1, "locks", "")
-		assertMakeName(`part_fish__x2y`, "fish", -2, "x2y", "")
-		assertParseName(`part_fish__x2y`, "fish", -1, "x2y", "")
-		assertMakeName(`part_fish_3_0004`, "fish", 2, "", "4")
-		assertParseName(`part_fish_4_0005`, "fish", 3, "", "0005")
-		assertMakeName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -3, "blkinfo", "jj5fvo3wr")
-		assertParseName(`part_fish__blkinfo_zz9fvo3wr`, "fish", -1, "blkinfo", "zz9fvo3wr")
-
-		// old-style temporary suffix (parse only)
-		assertParseName(`part_fish_4..tmp_0000000011`, "fish", 3, "", "000b")
-		assertParseName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -1, "blkinfo", "jj5fvo3wr")
-	}
-
-	// prepare format for long tests
-	assertFormat(`*.chunk.###`, `%s.chunk.%03d`, `%s.chunk._%s`, `^(.+?)\.chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	f.opt.StartFrom = 2
-
-	// valid data chunks
-	assertMakeName(`fish.chunk.003`, "fish", 1, "", "")
-	assertParseName(`fish.chunk.003`, "fish", 1, "", "")
-	assertMakeName(`fish.chunk.021`, "fish", 19, "", "")
-	assertParseName(`fish.chunk.021`, "fish", 19, "", "")
-
-	// valid temporary data chunks
-	assertMakeName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
-	assertParseName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
-	assertMakeName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
-	assertParseName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
-	assertMakeName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
-	assertParseName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
-	assertMakeName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
-	assertParseName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
-
-	// valid temporary data chunks (old temporary suffix, only parse)
-	assertParseName(`fish.chunk.004..tmp_0000000047`, "fish", 2, "", "001b")
-	assertParseName(`fish.chunk.323..tmp_9994567890123`, "fish", 321, "", "3jjfvo3wr")
-
-	// parsing invalid data chunk names
-	assertParseName(`fish.chunk.3`, "", -1, "", "")
-	assertParseName(`fish.chunk.001`, "", -1, "", "")
-	assertParseName(`fish.chunk.21`, "", -1, "", "")
-	assertParseName(`fish.chunk.-21`, "", -1, "", "")
-
-	assertParseName(`fish.chunk.004abcd`, "", -1, "", "")        // missing underscore delimiter
-	assertParseName(`fish.chunk.004__1234`, "", -1, "", "")      // extra underscore delimiter
-	assertParseName(`fish.chunk.004_123`, "", -1, "", "")        // too short temporary suffix
-	assertParseName(`fish.chunk.004_1234567890`, "", -1, "", "") // too long temporary suffix
-	assertParseName(`fish.chunk.004_-1234`, "", -1, "", "")      // temporary suffix must be positive
-	assertParseName(`fish.chunk.004_123E`, "", -1, "", "")       // uppercase not allowed
-	assertParseName(`fish.chunk.004_12.3`, "", -1, "", "")       // punctuation not allowed
-
-	// parsing invalid data chunk names (old temporary suffix)
-	assertParseName(`fish.chunk.004.tmp_0000000021`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_012345678901234567890123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk.323..tmp_12345678901234`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_-1`, "", -1, "", "")
-
-	// valid control chunks
-	assertMakeName(`fish.chunk._info`, "fish", -1, "info", "")
-	assertMakeName(`fish.chunk._locks`, "fish", -2, "locks", "")
-	assertMakeName(`fish.chunk._blkinfo`, "fish", -3, "blkinfo", "")
-	assertMakeName(`fish.chunk._x2y`, "fish", -4, "x2y", "")
-
-	assertParseName(`fish.chunk._info`, "fish", -1, "info", "")
-	assertParseName(`fish.chunk._locks`, "fish", -1, "locks", "")
-	assertParseName(`fish.chunk._blkinfo`, "fish", -1, "blkinfo", "")
-	assertParseName(`fish.chunk._x2y`, "fish", -1, "x2y", "")
-
-	// valid temporary control chunks
-	assertMakeName(`fish.chunk._info_0001`, "fish", -1, "info", "1")
-	assertMakeName(`fish.chunk._locks_4321`, "fish", -2, "locks", "4321")
-	assertMakeName(`fish.chunk._uploads_abcd`, "fish", -3, "uploads", "abcd")
-	assertMakeName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -4, "blkinfo", "xyzabcdef")
-	assertMakeName(`fish.chunk._x2y_1aaa`, "fish", -5, "x2y", "1aaa")
-
-	assertParseName(`fish.chunk._info_0001`, "fish", -1, "info", "0001")
-	assertParseName(`fish.chunk._locks_4321`, "fish", -1, "locks", "4321")
-	assertParseName(`fish.chunk._uploads_9abc`, "fish", -1, "uploads", "9abc")
-	assertParseName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -1, "blkinfo", "xyzabcdef")
-	assertParseName(`fish.chunk._x2y_1aaa`, "fish", -1, "x2y", "1aaa")
-
-	// valid temporary control chunks (old temporary suffix, parse only)
-	assertParseName(`fish.chunk._info..tmp_0000000047`, "fish", -1, "info", "001b")
-	assertParseName(`fish.chunk._locks..tmp_0000054321`, "fish", -1, "locks", "15wx")
-	assertParseName(`fish.chunk._uploads..tmp_0000000000`, "fish", -1, "uploads", "0000")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123`, "fish", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._x2y..tmp_0000000000`, "fish", -1, "x2y", "0000")
-
-	// parsing invalid control chunk names
-	assertParseName(`fish.chunk.metadata`, "", -1, "", "") // must be prepended by underscore
-	assertParseName(`fish.chunk.info`, "", -1, "", "")
-	assertParseName(`fish.chunk.locks`, "", -1, "", "")
-	assertParseName(`fish.chunk.uploads`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._os`, "", -1, "", "")        // too short
-	assertParseName(`fish.chunk._metadata`, "", -1, "", "")  // too long
-	assertParseName(`fish.chunk._blockinfo`, "", -1, "", "") // way too long
-	assertParseName(`fish.chunk._4me`, "", -1, "", "")       // cannot start with digit
-	assertParseName(`fish.chunk._567`, "", -1, "", "")       // cannot be all digits
-	assertParseName(`fish.chunk._me_ta`, "", -1, "", "")     // punctuation not allowed
-	assertParseName(`fish.chunk._in-fo`, "", -1, "", "")
-	assertParseName(`fish.chunk._.bin`, "", -1, "", "")
-	assertParseName(`fish.chunk._.2xy`, "", -1, "", "")
-
-	// parsing invalid temporary control chunks
-	assertParseName(`fish.chunk._blkinfo1234`, "", -1, "", "")     // missing underscore delimiter
-	assertParseName(`fish.chunk._info__1234`, "", -1, "", "")      // extra underscore delimiter
-	assertParseName(`fish.chunk._info_123`, "", -1, "", "")        // too short temporary suffix
-	assertParseName(`fish.chunk._info_1234567890`, "", -1, "", "") // too long temporary suffix
-	assertParseName(`fish.chunk._info_-1234`, "", -1, "", "")      // temporary suffix must be positive
-	assertParseName(`fish.chunk._info_123E`, "", -1, "", "")       // uppercase not allowed
-	assertParseName(`fish.chunk._info_12.3`, "", -1, "", "")       // punctuation not allowed
-
-	assertParseName(`fish.chunk._locks..tmp_123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk._meta..tmp_-1`, "", -1, "", "")
-	assertParseName(`fish.chunk._blockinfo..tmp_012345678901234567890123456789`, "", -1, "", "")
-
-	// short control chunk names: 3 letters ok, 1-2 letters not allowed
-	assertMakeName(`fish.chunk._ext`, "fish", -1, "ext", "")
-	assertParseName(`fish.chunk._int`, "fish", -1, "int", "")
-
-	assertMakeNamePanics("fish", -1, "in", "")
-	assertMakeNamePanics("fish", -1, "up", "4")
-	assertMakeNamePanics("fish", -1, "x", "")
-	assertMakeNamePanics("fish", -1, "c", "1z")
-
-	assertMakeName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0")
-	assertMakeName(`fish.chunk._ext_0026`, "fish", -1, "ext", "26")
-	assertMakeName(`fish.chunk._int_0abc`, "fish", -1, "int", "abc")
-	assertMakeName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
-	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-
-	assertParseName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0000")
-	assertParseName(`fish.chunk._ext_0026`, "fish", -1, "ext", "0026")
-	assertParseName(`fish.chunk._int_0abc`, "fish", -1, "int", "0abc")
-	assertParseName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
-	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-
-	// base file name can sometimes look like a valid chunk name
-	assertParseName(`fish.chunk.003.chunk.004`, "fish.chunk.003", 2, "", "")
-	assertParseName(`fish.chunk.003.chunk._info`, "fish.chunk.003", -1, "info", "")
-	assertParseName(`fish.chunk.003.chunk._Meta`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._info.chunk.004`, "fish.chunk._info", 2, "", "")
-	assertParseName(`fish.chunk._info.chunk._info`, "fish.chunk._info", -1, "info", "")
-	assertParseName(`fish.chunk._info.chunk._info.chunk._Meta`, "", -1, "", "")
-
-	// base file name looking like a valid chunk name (old temporary suffix)
-	assertParseName(`fish.chunk.003.chunk.005..tmp_0000000022`, "fish.chunk.003", 3, "", "000m")
-	assertParseName(`fish.chunk.003.chunk._x..tmp_0000054321`, "", -1, "", "")
-	assertParseName(`fish.chunk._info.chunk.005..tmp_0000000023`, "fish.chunk._info", 3, "", "000n")
-	assertParseName(`fish.chunk._info.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk.003.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.003", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._info.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._info", -1, "blkinfo", "3jjfvo3wr")
-
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.004`, "fish.chunk.004..tmp_0000000021", 2, "", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.005..tmp_0000000025`, "fish.chunk.004..tmp_0000000021", 3, "", "000p")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._info`, "fish.chunk.004..tmp_0000000021", -1, "info", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.004..tmp_0000000021", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.004`, "fish.chunk._blkinfo..tmp_9994567890123", 2, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.005..tmp_0000000026`, "fish.chunk._blkinfo..tmp_9994567890123", 3, "", "000q")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "info", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.004`, "fish.chunk._blkinfo..tmp_1234567890123456789", 2, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.005..tmp_0000000022`, "fish.chunk._blkinfo..tmp_1234567890123456789", 3, "", "000m")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "info", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	// attempts to make invalid chunk names
-	assertMakeNamePanics("fish", -1, "", "")          // neither data nor control
-	assertMakeNamePanics("fish", 0, "info", "")       // both data and control
-	assertMakeNamePanics("fish", -1, "metadata", "")  // control type too long
-	assertMakeNamePanics("fish", -1, "blockinfo", "") // control type way too long
-	assertMakeNamePanics("fish", -1, "2xy", "")       // first digit not allowed
-	assertMakeNamePanics("fish", -1, "123", "")       // all digits not allowed
-	assertMakeNamePanics("fish", -1, "Meta", "")      // only lower case letters allowed
-	assertMakeNamePanics("fish", -1, "in-fo", "")     // punctuation not allowed
-	assertMakeNamePanics("fish", -1, "_info", "")
-	assertMakeNamePanics("fish", -1, "info_", "")
-	assertMakeNamePanics("fish", -2, ".bind", "")
-	assertMakeNamePanics("fish", -2, "bind.", "")
-
-	assertMakeNamePanics("fish", -1, "", "1")          // neither data nor control
-	assertMakeNamePanics("fish", 0, "info", "23")      // both data and control
-	assertMakeNamePanics("fish", -1, "metadata", "45") // control type too long
-	assertMakeNamePanics("fish", -1, "blockinfo", "7") // control type way too long
-	assertMakeNamePanics("fish", -1, "2xy", "abc")     // first digit not allowed
-	assertMakeNamePanics("fish", -1, "123", "def")     // all digits not allowed
-	assertMakeNamePanics("fish", -1, "Meta", "mnk")    // only lower case letters allowed
-	assertMakeNamePanics("fish", -1, "in-fo", "xyz")   // punctuation not allowed
-	assertMakeNamePanics("fish", -1, "_info", "5678")
-	assertMakeNamePanics("fish", -1, "info_", "999")
-	assertMakeNamePanics("fish", -2, ".bind", "0")
-	assertMakeNamePanics("fish", -2, "bind.", "0")
-
-	assertMakeNamePanics("fish", 0, "", "1234567890") // temporary suffix too long
-	assertMakeNamePanics("fish", 0, "", "123F4")      // uppercase not allowed
-	assertMakeNamePanics("fish", 0, "", "123.")       // punctuation not allowed
-	assertMakeNamePanics("fish", 0, "", "_123")
-}
-
-func testSmallFileInternals(t *testing.T, f *Fs) {
-	const dir = "small"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = false
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-
-	checkSmallFileInternals := func(obj fs.Object) {
-		assert.NotNil(t, obj)
-		o, ok := obj.(*Object)
-		assert.True(t, ok)
-		assert.NotNil(t, o)
-		if o == nil {
-			return
-		}
-		switch {
-		case !f.useMeta:
-			// If meta format is "none", non-chunked file (even empty)
-			// internally is a single chunk without meta object.
-			assert.Nil(t, o.main)
-			assert.True(t, o.isComposite()) // sorry, sometimes a name is misleading
-			assert.Equal(t, 1, len(o.chunks))
-		case f.hashAll:
-			// Consistent hashing forces meta object on small files too
-			assert.NotNil(t, o.main)
-			assert.True(t, o.isComposite())
-			assert.Equal(t, 1, len(o.chunks))
-		default:
-			// normally non-chunked file is kept in the Object's main field
-			assert.NotNil(t, o.main)
-			assert.False(t, o.isComposite())
-			assert.Equal(t, 0, len(o.chunks))
-		}
-	}
-
-	checkContents := func(obj fs.Object, contents string) {
-		assert.NotNil(t, obj)
-		assert.Equal(t, int64(len(contents)), obj.Size())
-
-		r, err := obj.Open(ctx)
-		assert.NoError(t, err)
-		assert.NotNil(t, r)
-		if r == nil {
-			return
-		}
-		data, err := ioutil.ReadAll(r)
-		assert.NoError(t, err)
-		assert.Equal(t, contents, string(data))
-		_ = r.Close()
-	}
-
-	checkHashsum := func(obj fs.Object) {
-		var ht hash.Type
-		switch {
-		case !f.hashAll:
-			return
-		case f.useMD5:
-			ht = hash.MD5
-		case f.useSHA1:
-			ht = hash.SHA1
-		default:
-			return
-		}
-		// even empty files must have hashsum in consistent mode
-		sum, err := obj.Hash(ctx, ht)
-		assert.NoError(t, err)
-		assert.NotEqual(t, sum, "")
-	}
-
-	checkSmallFile := func(name, contents string) {
-		filename := path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
-		assert.NotNil(t, put)
-		checkSmallFileInternals(put)
-		checkContents(put, contents)
-		checkHashsum(put)
-
-		// objects returned by Put and NewObject must have similar structure
-		obj, err := f.NewObject(ctx, filename)
-		assert.NoError(t, err)
-		assert.NotNil(t, obj)
-		checkSmallFileInternals(obj)
-		checkContents(obj, contents)
-		checkHashsum(obj)
-
-		_ = obj.Remove(ctx)
-		_ = put.Remove(ctx) // for good
-	}
-
-	checkSmallFile("emptyfile", "")
-	checkSmallFile("smallfile", "Ok")
-}
-
-func testPreventCorruption(t *testing.T, f *Fs) {
-	if f.opt.ChunkSize > 50 {
-		t.Skip("this test requires small chunks")
-	}
-	const dir = "corrupted"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = true
-
-	contents := random.String(250)
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	const overlapMessage = "chunk overlap"
-
-	assertOverlapError := func(err error) {
-		assert.Error(t, err)
-		if err != nil {
-			assert.Contains(t, err.Error(), overlapMessage)
-		}
-	}
-
-	newFile := func(name string) fs.Object {
-		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		return obj
-	}
-	billyObj := newFile("billy")
-	billyTxn := billyObj.(*Object).xactID
-	if f.useNoRename {
-		require.True(t, billyTxn != "")
-	} else {
-		require.True(t, billyTxn == "")
-	}
-
-	billyChunkName := func(chunkNo int) string {
-		return f.makeChunkName(billyObj.Remote(), chunkNo, "", billyTxn)
-	}
-
-	err := f.Mkdir(ctx, billyChunkName(1))
-	assertOverlapError(err)
-
-	_, err = f.Move(ctx, newFile("silly1"), billyChunkName(2))
-	assert.Error(t, err)
-	assert.True(t, err == fs.ErrorCantMove || (err != nil && strings.Contains(err.Error(), overlapMessage)))
-
-	_, err = f.Copy(ctx, newFile("silly2"), billyChunkName(3))
-	assert.Error(t, err)
-	assert.True(t, err == fs.ErrorCantCopy || (err != nil && strings.Contains(err.Error(), overlapMessage)))
-
-	// accessing chunks in strict mode is prohibited
-	f.opt.FailHard = true
-	billyChunk4Name := billyChunkName(4)
-	_, err = f.base.NewObject(ctx, billyChunk4Name)
-	require.NoError(t, err)
-	_, err = f.NewObject(ctx, billyChunk4Name)
-	assertOverlapError(err)
-
-	f.opt.FailHard = false
-	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
-	assert.NoError(t, err)
-	require.NotNil(t, billyChunk4)
-
-	f.opt.FailHard = true
-	_, err = f.Put(ctx, bytes.NewBufferString(contents), billyChunk4)
-	assertOverlapError(err)
-
-	// you can freely read chunks (if you have an object)
-	r, err := billyChunk4.Open(ctx)
-	assert.NoError(t, err)
-	var chunkContents []byte
-	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
-		_ = r.Close()
-	})
-	assert.NoError(t, err)
-	assert.NotEqual(t, contents, string(chunkContents))
-
-	// but you can't change them
-	err = billyChunk4.Update(ctx, bytes.NewBufferString(contents), newFile("silly3"))
-	assertOverlapError(err)
-
-	// Remove isn't special, you can't corrupt files even if you have an object
-	err = billyChunk4.Remove(ctx)
-	assertOverlapError(err)
-
-	// recreate billy in case it was anyhow corrupted
-	willyObj := newFile("willy")
-	willyTxn := willyObj.(*Object).xactID
-	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", willyTxn)
-	f.opt.FailHard = false
-	willyChunk, err := f.NewObject(ctx, willyChunkName)
-	f.opt.FailHard = true
-	assert.NoError(t, err)
-	require.NotNil(t, willyChunk)
-
-	_, err = operations.Copy(ctx, f, willyChunk, willyChunkName, newFile("silly4"))
-	assertOverlapError(err)
-
-	// operations.Move will return error when chunker's Move refused
-	// to corrupt target file, but reverts to copy/delete method
-	// still trying to delete target chunk. Chunker must come to rescue.
-	_, err = operations.Move(ctx, f, willyChunk, willyChunkName, newFile("silly5"))
-	assertOverlapError(err)
-	r, err = willyChunk.Open(ctx)
-	assert.NoError(t, err)
-	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
-		_ = r.Close()
-	})
-	assert.NoError(t, err)
-}
-
-func testChunkNumberOverflow(t *testing.T, f *Fs) {
-	if f.opt.ChunkSize > 50 {
-		t.Skip("this test requires small chunks")
-	}
-	const dir = "wreaked"
-	const wreakNumber = 10200300
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	contents := random.String(100)
-
-	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
-		filename = path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
-			txnID = chunkObj.xactID
-		}
-		return
-	}
-
-	f.opt.FailHard = false
-	file, fileName, fileTxn := newFile(f, "wreaker")
-	wreak, _, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", fileTxn))
-
-	f.opt.FailHard = false
-	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
-	_, err := f.NewObject(ctx, fileName)
-	assert.Error(t, err)
-
-	f.opt.FailHard = true
-	_, err = f.List(ctx, dir)
-	assert.Error(t, err)
-	_, err = f.NewObject(ctx, fileName)
-	assert.Error(t, err)
-
-	f.opt.FailHard = false
-	_ = wreak.Remove(ctx)
-	_ = file.Remove(ctx)
-}
-
-func testMetadataInput(t *testing.T, f *Fs) {
-	const minChunkForTest = 50
-	if f.opt.ChunkSize < minChunkForTest {
-		t.Skip("this test requires chunks that fit metadata")
-	}
-
-	const dir = "usermeta"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = false
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-
-	putFile := func(f fs.Fs, name, contents, message string, check bool) fs.Object {
-		item := fstest.Item{Path: name, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
-		assert.NotNil(t, obj, message)
-		return obj
-	}
-
-	runSubtest := func(contents, name string) {
-		description := fmt.Sprintf("file with %s metadata", name)
-		filename := path.Join(dir, name)
-		require.True(t, len(contents) > 2 && len(contents) < minChunkForTest, description+" test data is correct")
-
-		part := putFile(f.base, f.makeChunkName(filename, 0, "", ""), "oops", "", true)
-		_ = putFile(f, filename, contents, "upload "+description, false)
-
-		obj, err := f.NewObject(ctx, filename)
-		assert.NoError(t, err, "access "+description)
-		assert.NotNil(t, obj)
-		assert.Equal(t, int64(len(contents)), obj.Size(), "size "+description)
-
-		o, ok := obj.(*Object)
-		assert.NotNil(t, ok)
-		if o != nil {
-			assert.True(t, o.isComposite() && len(o.chunks) == 1, description+" is forced composite")
-			o = nil
-		}
-
-		defer func() {
-			_ = obj.Remove(ctx)
-			_ = part.Remove(ctx)
-		}()
-
-		r, err := obj.Open(ctx)
-		assert.NoError(t, err, "open "+description)
-		assert.NotNil(t, r, "open stream of "+description)
-		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
-			assert.NoError(t, err, "read all of "+description)
-			assert.Equal(t, contents, string(data), description+" contents is ok")
-			_ = r.Close()
-		}
-	}
-
-	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "", "")
-	require.NoError(t, err)
-	todaysMeta := string(metaData)
-	runSubtest(todaysMeta, "today")
-
-	pastMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":1`)
-	pastMeta = regexp.MustCompile(`"size":[0-9]+`).ReplaceAllLiteralString(pastMeta, `"size":0`)
-	runSubtest(pastMeta, "past")
-
-	futureMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":999`)
-	futureMeta = regexp.MustCompile(`"nchunks":[0-9]+`).ReplaceAllLiteralString(futureMeta, `"nchunks":0,"x":"y"`)
-	runSubtest(futureMeta, "future")
-}
-
-// Test that chunker refuses to change on objects with future/unknown metadata
-func testFutureProof(t *testing.T, f *Fs) {
-	if f.opt.MetaFormat == "none" {
-		t.Skip("this test requires metadata support")
-	}
-
-	saveOpt := f.opt
-	ctx := context.Background()
-	f.opt.FailHard = true
-	const dir = "future"
-	const file = dir + "/test"
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	putPart := func(name string, part int, data, msg string) {
-		if part > 0 {
-			name = f.makeChunkName(name, part-1, "", "")
-		}
-		item := fstest.Item{Path: name, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
-		assert.NotNil(t, obj, msg)
-	}
-
-	// simulate chunked object from future
-	meta := `{"ver":999,"nchunks":3,"size":9,"garbage":"litter","sha1":"0707f2970043f9f7c22029482db27733deaec029"}`
-	putPart(file, 0, meta, "metaobject")
-	putPart(file, 1, "abc", "chunk1")
-	putPart(file, 2, "def", "chunk2")
-	putPart(file, 3, "ghi", "chunk3")
-
-	// List should succeed
-	ls, err := f.List(ctx, dir)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ls))
-	assert.Equal(t, int64(9), ls[0].Size())
-
-	// NewObject should succeed
-	obj, err := f.NewObject(ctx, file)
-	assert.NoError(t, err)
-	assert.Equal(t, file, obj.Remote())
-	assert.Equal(t, int64(9), obj.Size())
-
-	// Hash must fail
-	_, err = obj.Hash(ctx, hash.SHA1)
-	assert.Equal(t, ErrMetaUnknown, err)
-
-	// Move must fail
-	mobj, err := operations.Move(ctx, f, nil, file+"2", obj)
-	assert.Nil(t, mobj)
-	assert.Error(t, err)
-	if err != nil {
-		assert.Contains(t, err.Error(), "please upgrade rclone")
-	}
-
-	// Put must fail
-	oi := object.NewStaticObjectInfo(file, modTime, 3, true, nil, nil)
-	buf := bytes.NewBufferString("abc")
-	_, err = f.Put(ctx, buf, oi)
-	assert.Error(t, err)
-
-	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
-	assert.Nil(t, robj)
-	assert.NotNil(t, err)
-	if err != nil {
-		assert.Contains(t, err.Error(), "please upgrade rclone")
-	}
-}
-
-// The newer method of doing transactions without renaming should still be able to correctly process chunks that were created with renaming
-// If you attempt to do the inverse, however, the data chunks will be ignored causing commands to perform incorrectly
-func testBackwardsCompatibility(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't do norename transactions without metadata")
-	}
-	const dir = "backcomp"
-	ctx := context.Background()
-	saveOpt := f.opt
-	saveUseNoRename := f.useNoRename
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-		f.useNoRename = saveUseNoRename
-	}()
-	f.opt.ChunkSize = fs.SizeSuffix(10)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	contents := random.String(250)
-	newFile := func(f fs.Fs, name string) (fs.Object, string) {
-		filename := path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		return obj, filename
-	}
-
-	f.opt.FailHard = false
-	f.useNoRename = false
-	file, fileName := newFile(f, "renamefile")
-
-	f.opt.FailHard = false
-	item := fstest.NewItem(fileName, contents, modTime)
-
-	var items []fstest.Item
-	items = append(items, item)
-
-	f.useNoRename = true
-	fstest.CheckListingWithRoot(t, f, dir, items, nil, f.Precision())
-	_, err := f.NewObject(ctx, fileName)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = true
-	_, err = f.List(ctx, dir)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = false
-	_ = file.Remove(ctx)
-}
-
-func testChunkerServerSideMove(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't test norename transactions without metadata")
-	}
-
-	ctx := context.Background()
-	const dir = "servermovetest"
-	subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), dir)
-
-	subFs1, err := fs.NewFs(ctx, subRemote+"/subdir1")
-	assert.NoError(t, err)
-	fs1, isChunkerFs := subFs1.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs1.useNoRename = false
-	fs1.opt.ChunkSize = fs.SizeSuffix(3)
-
-	subFs2, err := fs.NewFs(ctx, subRemote+"/subdir2")
-	assert.NoError(t, err)
-	fs2, isChunkerFs := subFs2.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs2.useNoRename = true
-	fs2.opt.ChunkSize = fs.SizeSuffix(3)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	item := fstest.Item{Path: "movefile", ModTime: modTime}
-	contents := "abcdef"
-	_, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
-
-	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
-	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(len(contents)), dstFile.Size())
-
-	r, err := dstFile.Open(ctx)
-	assert.NoError(t, err)
-	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
-	assert.NoError(t, err)
-	assert.Equal(t, contents, string(data))
-	_ = r.Close()
-	_ = operations.Purge(ctx, f.base, dir)
-}
-
-// InternalTest dispatches all internal tests
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("PutLarge", func(t *testing.T) {
-		if *UploadKilobytes <= 0 {
-			t.Skip("-upload-kilobytes is not set")
-		}
-		testPutLarge(t, f, *UploadKilobytes)
-	})
-	t.Run("ChunkNameFormat", func(t *testing.T) {
-		testChunkNameFormat(t, f)
-	})
-	t.Run("SmallFileInternals", func(t *testing.T) {
-		testSmallFileInternals(t, f)
-	})
-	t.Run("PreventCorruption", func(t *testing.T) {
-		testPreventCorruption(t, f)
-	})
-	t.Run("ChunkNumberOverflow", func(t *testing.T) {
-		testChunkNumberOverflow(t, f)
-	})
-	t.Run("MetadataInput", func(t *testing.T) {
-		testMetadataInput(t, f)
-	})
-	t.Run("FutureProof", func(t *testing.T) {
-		testFutureProof(t, f)
-	})
-	t.Run("BackwardsCompatibility", func(t *testing.T) {
-		testBackwardsCompatibility(t, f)
-	})
-	t.Run("ChunkerServerSideMove", func(t *testing.T) {
-		testChunkerServerSideMove(t, f)
-	})
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -1,58 +0,0 @@
-// Test the Chunker filesystem interface
-package chunker_test
-
-import (
-	"flag"
-	"os"
-	"path/filepath"
-	"testing"
-
-	_ "github.com/rclone/rclone/backend/all" // for integration tests
-	"github.com/rclone/rclone/backend/chunker"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// Command line flags
-var (
-	// Invalid characters are not supported by some remotes, e.g. Mailru.
-	// We enable testing with invalid characters when -remote is not set, so
-	// chunker overlays a local directory, but invalid characters are disabled
-	// by default when -remote is set, e.g. when test_all runs backend tests.
-	// You can still test with invalid characters using the below flag.
-	UseBadChars = flag.Bool("bad-chars", false, "Set to test bad characters in file names when -remote is set")
-)
-
-// TestIntegration runs integration tests against a concrete remote
-// set by the -remote flag. If the flag is not set, it creates a
-// dynamic chunker overlay wrapping a local temporary directory.
-func TestIntegration(t *testing.T) {
-	opt := fstests.Opt{
-		RemoteName:               *fstest.RemoteName,
-		NilObject:                (*chunker.Object)(nil),
-		SkipBadWindowsCharacters: !*UseBadChars,
-		UnimplementableObjectMethods: []string{
-			"MimeType",
-			"GetTier",
-			"SetTier",
-		},
-		UnimplementableFsMethods: []string{
-			"PublicLink",
-			"OpenWriterAt",
-			"MergeDirs",
-			"DirCacheFlush",
-			"UserInfo",
-			"Disconnect",
-		},
-	}
-	if *fstest.RemoteName == "" {
-		name := "TestChunker"
-		opt.RemoteName = name + ":"
-		tempDir := filepath.Join(os.TempDir(), "rclone-chunker-test-standard")
-		opt.ExtraConfig = []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "chunker"},
-			{Name: name, Key: "remote", Value: tempDir},
-		}
-	}
-	fstests.Run(t, &opt)
-}

backend/compress/.gitignore

@@ -1 +0,0 @@
-test

backend/compress/compress_test.go

@@ -1,65 +0,0 @@
-// Test Crypt filesystem interface
-package compress
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	_ "github.com/rclone/rclone/backend/drive"
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/s3"
-	_ "github.com/rclone/rclone/backend/swift"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	opt := fstests.Opt{
-		RemoteName: *fstest.RemoteName,
-		NilObject:  (*Object)(nil),
-		UnimplementableFsMethods: []string{
-			"OpenWriterAt",
-			"MergeDirs",
-			"DirCacheFlush",
-			"PutUnchecked",
-			"PutStream",
-			"UserInfo",
-			"Disconnect",
-		},
-		TiersToTest:                  []string{"STANDARD", "STANDARD_IA"},
-		UnimplementableObjectMethods: []string{}}
-	fstests.Run(t, &opt)
-}
-
-// TestRemoteGzip tests GZIP compression
-func TestRemoteGzip(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-gzip")
-	name := "TestCompressGzip"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*Object)(nil),
-		UnimplementableFsMethods: []string{
-			"OpenWriterAt",
-			"MergeDirs",
-			"DirCacheFlush",
-			"PutUnchecked",
-			"PutStream",
-			"UserInfo",
-			"Disconnect",
-		},
-		UnimplementableObjectMethods: []string{
-			"GetTier",
-			"SetTier",
-		},
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "compress"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "compression_mode", Value: "gzip"},
-		},
-	})
-}

backend/crypt/cipher.go

@@ -2,7 +2,6 @@ package crypt
 
 import (
 	"bytes"
-	"context"
 	"crypto/aes"
 	gocipher "crypto/cipher"
 	"crypto/rand"
@@ -12,17 +11,17 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"unicode/utf8"
 
+	"github.com/ncw/rclone/backend/crypt/pkcs7"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/backend/crypt/pkcs7"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/version"
-	"github.com/rfjakob/eme"
+
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/crypto/scrypt"
+
+	"github.com/rfjakob/eme"
 )
 
 // Constants
@@ -44,7 +43,6 @@ var (
 	ErrorBadDecryptControlChar   = errors.New("bad decryption - contains control chars")
 	ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize")
 	ErrorTooShortAfterDecode     = errors.New("too short after base32 decode")
-	ErrorTooLongAfterDecode      = errors.New("too long after base32 decode")
 	ErrorEncryptedFileTooShort   = errors.New("file is too short to be encrypted")
 	ErrorEncryptedFileBadHeader  = errors.New("file has truncated block header")
 	ErrorEncryptedBadMagic       = errors.New("not an encrypted file - bad magic string")
@@ -71,7 +69,31 @@ type ReadSeekCloser interface {
 }
 
 // OpenRangeSeek opens the file handle at the offset with the limit given
-type OpenRangeSeek func(ctx context.Context, offset, limit int64) (io.ReadCloser, error)
+type OpenRangeSeek func(offset, limit int64) (io.ReadCloser, error)
+
+// Cipher is used to swap out the encryption implementations
+type Cipher interface {
+	// EncryptFileName encrypts a file path
+	EncryptFileName(string) string
+	// DecryptFileName decrypts a file path, returns error if decrypt was invalid
+	DecryptFileName(string) (string, error)
+	// EncryptDirName encrypts a directory path
+	EncryptDirName(string) string
+	// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
+	DecryptDirName(string) (string, error)
+	// EncryptData
+	EncryptData(io.Reader) (io.Reader, error)
+	// DecryptData
+	DecryptData(io.ReadCloser) (io.ReadCloser, error)
+	// DecryptDataSeek decrypt at a given position
+	DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error)
+	// EncryptedSize calculates the size of the data when encrypted
+	EncryptedSize(int64) int64
+	// DecryptedSize calculates the size of the data when decrypted
+	DecryptedSize(int64) (int64, error)
+	// NameEncryptionMode returns the used mode for name handling
+	NameEncryptionMode() NameEncryptionMode
+}
 
 // NameEncryptionMode is the type of file name encryption in use
 type NameEncryptionMode int
@@ -114,8 +136,7 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-// Cipher defines an encoding and decoding cipher for the crypt backend
-type Cipher struct {
+type cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
 	nameKey        [32]byte                  // 16,24 or 32 bytes
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
@@ -127,8 +148,8 @@ type Cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
-	c := &Cipher{
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*cipher, error) {
+	c := &cipher{
 		mode:           mode,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
@@ -149,9 +170,9 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 // If salt is "" we use a fixed salt just to make attackers lives
 // slighty harder than using no salt.
 //
-// Note that empty password makes all 0x00 keys which is used in the
+// Note that empty passsword makes all 0x00 keys which is used in the
 // tests.
-func (c *Cipher) Key(password, salt string) (err error) {
+func (c *cipher) Key(password, salt string) (err error) {
 	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
 	var saltBytes = defaultSalt
 	if salt != "" {
@@ -175,18 +196,33 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() []byte {
+func (c *cipher) getBlock() []byte {
 	return c.buffers.Get().([]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf []byte) {
+func (c *cipher) putBlock(buf []byte) {
 	if len(buf) != blockSize {
 		panic("bad blocksize returned to pool")
 	}
 	c.buffers.Put(buf)
 }
 
+// check to see if the byte string is valid with no control characters
+// from 0x00 to 0x1F and is a valid UTF-8 string
+func checkValidString(buf []byte) error {
+	for i := range buf {
+		c := buf[i]
+		if c >= 0x00 && c < 0x20 || c == 0x7F {
+			return ErrorBadDecryptControlChar
+		}
+	}
+	if !utf8.Valid(buf) {
+		return ErrorBadDecryptUTF8
+	}
+	return nil
+}
+
 // encodeFileName encodes a filename using a modified version of
 // standard base32 as described in RFC4648
 //
@@ -219,13 +255,13 @@ func decodeFileName(in string) ([]byte, error) {
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
 // Rogaway.
 //
-// This makes for deterministic encryption which is what we want - the
+// This makes for determinstic encryption which is what we want - the
 // same filename must encrypt to the same thing.
 //
 // This means that
 //  * filenames with the same name will encrypt the same
 //  * filenames which start the same won't have a common prefix
-func (c *Cipher) encryptSegment(plaintext string) string {
+func (c *cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -235,7 +271,7 @@ func (c *Cipher) encryptSegment(plaintext string) string {
 }
 
 // decryptSegment decrypts a path segment
-func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
+func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -250,19 +286,20 @@ func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 		// not possible if decodeFilename() working correctly
 		return "", ErrorTooShortAfterDecode
 	}
-	if len(rawCiphertext) > 2048 {
-		return "", ErrorTooLongAfterDecode
-	}
 	paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt)
 	plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext)
 	if err != nil {
 		return "", err
 	}
+	err = checkValidString(plaintext)
+	if err != nil {
+		return "", err
+	}
 	return string(plaintext), err
 }
 
 // Simple obfuscation routines
-func (c *Cipher) obfuscateSegment(plaintext string) string {
+func (c *cipher) obfuscateSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -349,7 +386,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	return result.String()
 }
 
-func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
+func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -424,7 +461,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(newRune)
+			_, _ = result.WriteRune(rune(newRune))
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -436,7 +473,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *Cipher) encryptFileName(in string) string {
+func (c *cipher) encryptFileName(in string) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -444,38 +481,17 @@ func (c *Cipher) encryptFileName(in string) string {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets encrypted/obfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i] = c.encryptSegment(segments[i])
 		} else {
 			segments[i] = c.obfuscateSegment(segments[i])
 		}
-
-		// Add back a version to the encrypted/obfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/")
 }
 
 // EncryptFileName encrypts a file path
-func (c *Cipher) EncryptFileName(in string) string {
+func (c *cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
 		return in + encryptedSuffix
 	}
@@ -483,7 +499,7 @@ func (c *Cipher) EncryptFileName(in string) string {
 }
 
 // EncryptDirName encrypts a directory path
-func (c *Cipher) EncryptDirName(in string) string {
+func (c *cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
@@ -491,7 +507,7 @@ func (c *Cipher) EncryptDirName(in string) string {
 }
 
 // decryptFileName decrypts a file path
-func (c *Cipher) decryptFileName(in string) (string, error) {
+func (c *cipher) decryptFileName(in string) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -500,21 +516,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets decrypted/deobfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i], err = c.decryptSegment(segments[i])
 		} else {
@@ -524,46 +525,31 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if err != nil {
 			return "", err
 		}
-
-		// Add back a version to the decrypted/deobfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/"), nil
 }
 
 // DecryptFileName decrypts a file path
-func (c *Cipher) DecryptFileName(in string) (string, error) {
+func (c *cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
 		remainingLength := len(in) - len(encryptedSuffix)
-		if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
-			return "", ErrorNotAnEncryptedFile
+		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
+			return in[:remainingLength], nil
 		}
-		decrypted := in[:remainingLength]
-		if version.Match(decrypted) {
-			_, unversioned := version.Remove(decrypted)
-			if unversioned == "" {
-				return "", ErrorNotAnEncryptedFile
-			}
-		}
-		// Leave the version string on, if it was there
-		return decrypted, nil
+		return "", ErrorNotAnEncryptedFile
 	}
 	return c.decryptFileName(in)
 }
 
 // DecryptDirName decrypts a directory path
-func (c *Cipher) DecryptDirName(in string) (string, error) {
+func (c *cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
 	return c.decryptFileName(in)
 }
 
-// NameEncryptionMode returns the encryption mode in use for names
-func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
+func (c *cipher) NameEncryptionMode() NameEncryptionMode {
 	return c.mode
 }
 
@@ -611,7 +597,7 @@ func (n *nonce) increment() {
 	n.carry(0)
 }
 
-// add a uint64 to the nonce
+// add an uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
 	for i := 0; i < 8; i++ {
@@ -631,7 +617,7 @@ func (n *nonce) add(x uint64) {
 type encrypter struct {
 	mu       sync.Mutex
 	in       io.Reader
-	c        *Cipher
+	c        *cipher
 	nonce    nonce
 	buf      []byte
 	readBuf  []byte
@@ -641,7 +627,7 @@ type encrypter struct {
 }
 
 // newEncrypter creates a new file handle encrypting on the fly
-func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
+func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 	fh := &encrypter{
 		in:      in,
 		c:       c,
@@ -685,8 +671,11 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 		}
 		// possibly err != nil here, but we will process the
 		// data and the next call to ReadFull will return 0, err
+		// Write nonce to start of block
+		copy(fh.buf, fh.nonce[:])
 		// Encrypt the block using the nonce
-		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		block := fh.buf
+		secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
@@ -710,19 +699,13 @@ func (fh *encrypter) finish(err error) (int, error) {
 }
 
 // Encrypt data encrypts the data stream
-func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
+func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
 	in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
 	out, err := c.newEncrypter(in, nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	return wrap(out), out, nil // and wrap the accounting back on
-}
-
-// EncryptData encrypts the data stream
-func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
-	out, _, err := c.encryptData(in)
-	return out, err
+	return wrap(out), nil // and wrap the accounting back on
 }
 
 // decrypter decrypts an io.ReaderCloser on the fly
@@ -731,7 +714,7 @@ type decrypter struct {
 	rc           io.ReadCloser
 	nonce        nonce
 	initialNonce nonce
-	c            *Cipher
+	c            *cipher
 	buf          []byte
 	readBuf      []byte
 	bufIndex     int
@@ -742,7 +725,7 @@ type decrypter struct {
 }
 
 // newDecrypter creates a new file handle decrypting on the fly
-func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
+func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	fh := &decrypter{
 		rc:      rc,
 		c:       c,
@@ -763,29 +746,29 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retrieve the nonce
+	// retreive the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil
 }
 
 // newDecrypterSeek creates a new file handle decrypting on the fly
-func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
+func (c *cipher) newDecrypterSeek(open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
 	var rc io.ReadCloser
 	doRangeSeek := false
 	setLimit := false
 	// Open initially with no seek
 	if offset == 0 && limit < 0 {
 		// If no offset or limit then open whole file
-		rc, err = open(ctx, 0, -1)
+		rc, err = open(0, -1)
 	} else if offset == 0 {
 		// If no offset open the header + limit worth of the file
 		_, underlyingLimit, _, _ := calculateUnderlying(offset, limit)
-		rc, err = open(ctx, 0, int64(fileHeaderSize)+underlyingLimit)
+		rc, err = open(0, int64(fileHeaderSize)+underlyingLimit)
 		setLimit = true
 	} else {
 		// Otherwise just read the header to start with
-		rc, err = open(ctx, 0, int64(fileHeaderSize))
+		rc, err = open(0, int64(fileHeaderSize))
 		doRangeSeek = true
 	}
 	if err != nil {
@@ -798,7 +781,7 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 	}
 	fh.open = open // will be called by fh.RangeSeek
 	if doRangeSeek {
-		_, err = fh.RangeSeek(ctx, offset, io.SeekStart, limit)
+		_, err = fh.RangeSeek(offset, io.SeekStart, limit)
 		if err != nil {
 			_ = fh.Close()
 			return nil, err
@@ -831,7 +814,8 @@ func (fh *decrypter) fillBuffer() (err error) {
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	block := fh.buf
+	_, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
 		if err != nil {
 			return err // return pending error as it is likely more accurate
@@ -917,7 +901,7 @@ func calculateUnderlying(offset, limit int64) (underlyingOffset, underlyingLimit
 // limiting the total length to limit.
 //
 // RangeSeek with a limit of < 0 is equivalent to a regular Seek.
-func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, limit int64) (int64, error) {
+func (fh *decrypter) RangeSeek(offset int64, whence int, limit int64) (int64, error) {
 	fh.mu.Lock()
 	defer fh.mu.Unlock()
 
@@ -944,7 +928,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 	// Can we seek underlying stream directly?
 	if do, ok := fh.rc.(fs.RangeSeeker); ok {
 		// Seek underlying stream directly
-		_, err := do.RangeSeek(ctx, underlyingOffset, 0, underlyingLimit)
+		_, err := do.RangeSeek(underlyingOffset, 0, underlyingLimit)
 		if err != nil {
 			return 0, fh.finish(err)
 		}
@@ -954,7 +938,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 		fh.rc = nil
 
 		// Re-open the underlying object with the offset given
-		rc, err := fh.open(ctx, underlyingOffset, underlyingLimit)
+		rc, err := fh.open(underlyingOffset, underlyingLimit)
 		if err != nil {
 			return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset and limit"))
 		}
@@ -983,7 +967,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 
 // Seek implements the io.Seeker interface
 func (fh *decrypter) Seek(offset int64, whence int) (int64, error) {
-	return fh.RangeSeek(context.TODO(), offset, whence, -1)
+	return fh.RangeSeek(offset, whence, -1)
 }
 
 // finish sets the final error and tidies up
@@ -1044,7 +1028,7 @@ func (fh *decrypter) finishAndClose(err error) error {
 }
 
 // DecryptData decrypts the data stream
-func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
+func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 	out, err := c.newDecrypter(rc)
 	if err != nil {
 		return nil, err
@@ -1057,8 +1041,8 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 // The open function must return a ReadCloser opened to the offset supplied
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
-func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
-	out, err := c.newDecrypterSeek(ctx, open, offset, limit)
+func (c *cipher) DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
+	out, err := c.newDecrypterSeek(open, offset, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -1066,7 +1050,7 @@ func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset
 }
 
 // EncryptedSize calculates the size of the data when encrypted
-func (c *Cipher) EncryptedSize(size int64) int64 {
+func (c *cipher) EncryptedSize(size int64) int64 {
 	blocks, residue := size/blockDataSize, size%blockDataSize
 	encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
 	if residue != 0 {
@@ -1076,7 +1060,7 @@ func (c *Cipher) EncryptedSize(size int64) int64 {
 }
 
 // DecryptedSize calculates the size of the data when decrypted
-func (c *Cipher) DecryptedSize(size int64) (int64, error) {
+func (c *cipher) DecryptedSize(size int64) (int64, error) {
 	size -= int64(fileHeaderSize)
 	if size < 0 {
 		return 0, ErrorEncryptedFileTooShort
@@ -1095,6 +1079,7 @@ func (c *Cipher) DecryptedSize(size int64) (int64, error) {
 
 // check interfaces
 var (
+	_ Cipher         = (*cipher)(nil)
 	_ io.ReadCloser  = (*decrypter)(nil)
 	_ io.Seeker      = (*decrypter)(nil)
 	_ fs.RangeSeeker = (*decrypter)(nil)

backend/crypt/cipher_test.go

@@ -2,7 +2,6 @@ package crypt
 
 import (
 	"bytes"
-	"context"
 	"encoding/base32"
 	"fmt"
 	"io"
@@ -10,9 +9,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ncw/rclone/backend/crypt/pkcs7"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/backend/crypt/pkcs7"
-	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -45,6 +43,69 @@ func TestNewNameEncryptionModeString(t *testing.T) {
 	assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3")
 }
 
+func TestValidString(t *testing.T) {
+	for _, test := range []struct {
+		in       string
+		expected error
+	}{
+		{"", nil},
+		{"\x01", ErrorBadDecryptControlChar},
+		{"a\x02", ErrorBadDecryptControlChar},
+		{"abc\x03", ErrorBadDecryptControlChar},
+		{"abc\x04def", ErrorBadDecryptControlChar},
+		{"\x05d", ErrorBadDecryptControlChar},
+		{"\x06def", ErrorBadDecryptControlChar},
+		{"\x07", ErrorBadDecryptControlChar},
+		{"\x08", ErrorBadDecryptControlChar},
+		{"\x09", ErrorBadDecryptControlChar},
+		{"\x0A", ErrorBadDecryptControlChar},
+		{"\x0B", ErrorBadDecryptControlChar},
+		{"\x0C", ErrorBadDecryptControlChar},
+		{"\x0D", ErrorBadDecryptControlChar},
+		{"\x0E", ErrorBadDecryptControlChar},
+		{"\x0F", ErrorBadDecryptControlChar},
+		{"\x10", ErrorBadDecryptControlChar},
+		{"\x11", ErrorBadDecryptControlChar},
+		{"\x12", ErrorBadDecryptControlChar},
+		{"\x13", ErrorBadDecryptControlChar},
+		{"\x14", ErrorBadDecryptControlChar},
+		{"\x15", ErrorBadDecryptControlChar},
+		{"\x16", ErrorBadDecryptControlChar},
+		{"\x17", ErrorBadDecryptControlChar},
+		{"\x18", ErrorBadDecryptControlChar},
+		{"\x19", ErrorBadDecryptControlChar},
+		{"\x1A", ErrorBadDecryptControlChar},
+		{"\x1B", ErrorBadDecryptControlChar},
+		{"\x1C", ErrorBadDecryptControlChar},
+		{"\x1D", ErrorBadDecryptControlChar},
+		{"\x1E", ErrorBadDecryptControlChar},
+		{"\x1F", ErrorBadDecryptControlChar},
+		{"\x20", nil},
+		{"\x7E", nil},
+		{"\x7F", ErrorBadDecryptControlChar},
+		{"£100", nil},
+		{`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil},
+		{"£100", nil},
+		// Following tests from https://secure.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
+		{"a", nil},                                        // Valid ASCII
+		{"\xc3\xb1", nil},                                 // Valid 2 Octet Sequence
+		{"\xc3\x28", ErrorBadDecryptUTF8},                 // Invalid 2 Octet Sequence
+		{"\xa0\xa1", ErrorBadDecryptUTF8},                 // Invalid Sequence Identifier
+		{"\xe2\x82\xa1", nil},                             // Valid 3 Octet Sequence
+		{"\xe2\x28\xa1", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 2nd Octet)
+		{"\xe2\x82\x28", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 3rd Octet)
+		{"\xf0\x90\x8c\xbc", nil},                         // Valid 4 Octet Sequence
+		{"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 2nd Octet)
+		{"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 3rd Octet)
+		{"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 4th Octet)
+		{"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8},     // Valid 5 Octet Sequence (but not Unicode!)
+		{"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!)
+	} {
+		actual := checkValidString([]byte(test.in))
+		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
+	}
+}
+
 func TestEncodeFileName(t *testing.T) {
 	for _, test := range []struct {
 		in       string
@@ -133,10 +194,6 @@ func TestEncryptSegment(t *testing.T) {
 
 func TestDecryptSegment(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
-	longName := make([]byte, 3328)
-	for i := range longName {
-		longName[i] = 'a'
-	}
 	c, _ := newCipher(NameEncryptionStandard, "", "", true)
 	for _, test := range []struct {
 		in          string
@@ -144,10 +201,11 @@ func TestDecryptSegment(t *testing.T) {
 	}{
 		{"64=", ErrorBadBase32Encoding},
 		{"!", base32.CorruptInputError(0)},
-		{string(longName), ErrorTooLongAfterDecode},
 		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+		{c.encryptSegment("\x01"), ErrorBadDecryptControlChar},
+		{c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8},
 	} {
 		actual, actualErr := c.decryptSegment(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
@@ -160,29 +218,22 @@ func TestEncryptFileName(t *testing.T) {
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Standard mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionStandard, "", "", false)
 	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
 	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
 	assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Now off mode
 	c, _ = newCipher(NameEncryptionOff, "", "", true)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
-	assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
-	assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 	// Obfuscation mode with directory name encryption off
 	c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
 	assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
-	assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 }
@@ -201,19 +252,14 @@ func TestDecryptFileName(t *testing.T) {
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
 		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
 		{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
 		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
 		{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
 		{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
 		{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
 		{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
 		{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
 		{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
 	} {
 		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
 		actual, actualErr := c.DecryptFileName(test.in)
@@ -653,16 +699,16 @@ var (
 
 // Test test infrastructure first!
 func TestRandomSource(t *testing.T) {
-	source := newRandomSource(1e8)
-	sink := newRandomSource(1e8)
+	source := newRandomSource(1E8)
+	sink := newRandomSource(1E8)
 	n, err := io.Copy(sink, source)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(1e8), n)
+	assert.Equal(t, int64(1E8), n)
 
-	source = newRandomSource(1e8)
+	source = newRandomSource(1E8)
 	buf := make([]byte, 16)
 	_, _ = source.Read(buf)
-	sink = newRandomSource(1e8)
+	sink = newRandomSource(1E8)
 	_, err = io.Copy(sink, source)
 	assert.Error(t, err, "Error in stream")
 }
@@ -702,23 +748,23 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 }
 
 func TestEncryptDecrypt1(t *testing.T) {
-	testEncryptDecrypt(t, 1, 1e7)
+	testEncryptDecrypt(t, 1, 1E7)
 }
 
 func TestEncryptDecrypt32(t *testing.T) {
-	testEncryptDecrypt(t, 32, 1e8)
+	testEncryptDecrypt(t, 32, 1E8)
 }
 
 func TestEncryptDecrypt4096(t *testing.T) {
-	testEncryptDecrypt(t, 4096, 1e8)
+	testEncryptDecrypt(t, 4096, 1E8)
 }
 
 func TestEncryptDecrypt65536(t *testing.T) {
-	testEncryptDecrypt(t, 65536, 1e8)
+	testEncryptDecrypt(t, 65536, 1E8)
 }
 
 func TestEncryptDecrypt65537(t *testing.T) {
-	testEncryptDecrypt(t, 65537, 1e8)
+	testEncryptDecrypt(t, 65537, 1E8)
 }
 
 var (
@@ -751,7 +797,7 @@ func TestEncryptData(t *testing.T) {
 	} {
 		c, err := newCipher(NameEncryptionStandard, "", "", true)
 		assert.NoError(t, err)
-		c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
+		c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
 		// Check encode works
 		buf := bytes.NewBuffer(test.in)
@@ -774,7 +820,7 @@ func TestEncryptData(t *testing.T) {
 func TestNewEncrypter(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
-	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
+	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
 	z := &zeroes{}
 
@@ -797,15 +843,23 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
+	in := &errorReader{io.ErrUnexpectedEOF}
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1E6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
 
+type errorReader struct {
+	err error
+}
+
+func (er errorReader) Read(p []byte) (n int, err error) {
+	return 0, er.err
+}
+
 type closeDetector struct {
 	io.Reader
 	closed int
@@ -825,7 +879,7 @@ func (c *closeDetector) Close() error {
 func TestNewDecrypter(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
-	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
+	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
 	cd := newCloseDetector(bytes.NewBuffer(file0))
 	fh, err := c.newDecrypter(cd)
@@ -843,7 +897,7 @@ func TestNewDecrypter(t *testing.T) {
 		assert.Equal(t, 1, cd.closed)
 	}
 
-	er := &readers.ErrorReader{Err: errors.New("potato")}
+	er := &errorReader{errors.New("potato")}
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
@@ -869,14 +923,14 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
+	in2 := &errorReader{io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
 	in := ioutil.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1E6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -906,7 +960,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Open stream with a seek of underlyingOffset
 	var reader io.ReadCloser
-	open := func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
+	open := func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
 		end := len(ciphertext)
 		if underlyingLimit >= 0 {
 			end = int(underlyingOffset + underlyingLimit)
@@ -941,13 +995,13 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 
-	// Now try decoding it with an open/seek
+	// Now try decoding it with a open/seek
 	for _, offset := range trials {
 		for _, limit := range limits {
 			if offset+limit > len(plaintext) {
 				continue
 			}
-			rc, err := c.DecryptDataSeek(context.Background(), open, int64(offset), int64(limit))
+			rc, err := c.DecryptDataSeek(open, int64(offset), int64(limit))
 			assert.NoError(t, err)
 
 			check(rc, offset, limit)
@@ -955,14 +1009,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 	}
 
 	// Try decoding it with a single open and lots of seeks
-	fh, err := c.DecryptDataSeek(context.Background(), open, 0, -1)
+	fh, err := c.DecryptDataSeek(open, 0, -1)
 	assert.NoError(t, err)
 	for _, offset := range trials {
 		for _, limit := range limits {
 			if offset+limit > len(plaintext) {
 				continue
 			}
-			_, err := fh.RangeSeek(context.Background(), int64(offset), io.SeekStart, int64(limit))
+			_, err := fh.RangeSeek(int64(offset), io.SeekStart, int64(limit))
 			assert.NoError(t, err)
 
 			check(fh, offset, limit)
@@ -1013,7 +1067,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 	} {
 		what := fmt.Sprintf("offset = %d, limit = %d", test.offset, test.limit)
 		callCount := 0
-		testOpen := func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
+		testOpen := func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
 			switch callCount {
 			case 0:
 				assert.Equal(t, int64(0), underlyingOffset, what)
@@ -1025,11 +1079,11 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				t.Errorf("Too many calls %d for %s", callCount+1, what)
 			}
 			callCount++
-			return open(ctx, underlyingOffset, underlyingLimit)
+			return open(underlyingOffset, underlyingLimit)
 		}
-		fh, err := c.DecryptDataSeek(context.Background(), testOpen, 0, -1)
+		fh, err := c.DecryptDataSeek(testOpen, 0, -1)
 		assert.NoError(t, err)
-		gotOffset, err := fh.RangeSeek(context.Background(), test.offset, io.SeekStart, test.limit)
+		gotOffset, err := fh.RangeSeek(test.offset, io.SeekStart, test.limit)
 		assert.NoError(t, err)
 		assert.Equal(t, gotOffset, test.offset)
 	}
@@ -1123,7 +1177,7 @@ func TestDecrypterRead(t *testing.T) {
 
 	// Test producing an error on the file on Read the underlying file
 	in1 := bytes.NewBuffer(file1)
-	in2 := &readers.ErrorReader{Err: errors.New("potato")}
+	in2 := &errorReader{errors.New("potato")}
 	in := io.MultiReader(in1, in2)
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)

backend/crypt/crypt.go

@@ -2,22 +2,19 @@
 package crypt
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"path"
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/hash"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fspath"
-	"github.com/rclone/rclone/fs/hash"
 )
 
 // Globals
@@ -27,10 +24,9 @@ func init() {
 		Name:        "crypt",
 		Description: "Encrypt/Decrypt a remote",
 		NewFs:       NewFs,
-		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
-			Help:     "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, e.g. \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
+			Help:     "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
 			Required: true,
 		}, {
 			Name:    "filename_encryption",
@@ -38,21 +34,19 @@ func init() {
 			Default: "standard",
 			Examples: []fs.OptionExample{
 				{
+					Value: "off",
+					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
+				}, {
 					Value: "standard",
 					Help:  "Encrypt the filenames see the docs for the details.",
 				}, {
 					Value: "obfuscate",
 					Help:  "Very simple filename obfuscation.",
-				}, {
-					Value: "off",
-					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
 				},
 			},
 		}, {
-			Name: "directory_name_encryption",
-			Help: `Option to either encrypt directory names or leave them intact.
-
-NB If filename_encryption is "off" then this option will do nothing.`,
+			Name:    "directory_name_encryption",
+			Help:    "Option to either encrypt directory names or leave them intact.",
 			Default: true,
 			Examples: []fs.OptionExample{
 				{
@@ -68,60 +62,22 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 			Name:       "password",
 			Help:       "Password or pass phrase for encryption.",
 			IsPassword: true,
-			Required:   true,
 		}, {
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
 		}, {
-			Name:    "server_side_across_configs",
-			Default: false,
-			Help: `Allow server-side operations (e.g. copy) to work across different crypt configs.
-
-Normally this option is not what you want, but if you have two crypts
-pointing to the same backend you can use it.
-
-This can be used, for example, to change file name encryption type
-without re-uploading all the data. Just make two crypt backends
-pointing to two different directories with the single changed
-parameter and use rclone move to move the files between the crypt
-remotes.`,
-			Advanced: true,
-		}, {
-			Name: "show_mapping",
-			Help: `For all files listed show how the names encrypt.
-
-If this flag is set then for each file that the remote is asked to
-list, it will log (at level INFO) a line stating the decrypted file
-name and the encrypted file name.
-
-This is so you can work out which encrypted names are which decrypted
-names just in case you need to do something with the encrypted file
-names, or for debugging purposes.`,
+			Name:     "show_mapping",
+			Help:     "For all files listed show how the names encrypt.",
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
-		}, {
-			Name:     "no_data_encryption",
-			Help:     "Option to either encrypt file data or leave it unencrypted.",
-			Default:  false,
-			Advanced: true,
-			Examples: []fs.OptionExample{
-				{
-					Value: "true",
-					Help:  "Don't encrypt file data, leave it unencrypted.",
-				},
-				{
-					Value: "false",
-					Help:  "Encrypt file data.",
-				},
-			},
 		}},
 	})
 }
 
 // newCipherForConfig constructs a Cipher for the given config name
-func newCipherForConfig(opt *Options) (*Cipher, error) {
+func newCipherForConfig(opt *Options) (Cipher, error) {
 	mode, err := NewNameEncryptionMode(opt.FilenameEncryption)
 	if err != nil {
 		return nil, err
@@ -148,7 +104,7 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 }
 
 // NewCipher constructs a Cipher for the given config
-func NewCipher(m configmap.Mapper) (*Cipher, error) {
+func NewCipher(m configmap.Mapper) (Cipher, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -158,8 +114,8 @@ func NewCipher(m configmap.Mapper) (*Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -174,25 +130,16 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	if strings.HasPrefix(remote, name+":") {
 		return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
 	}
-	// Make sure to remove trailing . referring to the current dir
-	if path.Base(rpath) == "." {
-		rpath = strings.TrimSuffix(rpath, ".")
-	}
 	// Look for a file first
-	var wrappedFs fs.Fs
-	if rpath == "" {
-		wrappedFs, err = cache.Get(ctx, remote)
-	} else {
-		remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath))
-		wrappedFs, err = cache.Get(ctx, remotePath)
-		// if that didn't produce a file, look for a directory
-		if err != fs.ErrorIsFile {
-			remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath))
-			wrappedFs, err = cache.Get(ctx, remotePath)
-		}
+	remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
+	wrappedFs, err := fs.NewFs(remotePath)
+	// if that didn't produce a file, look for a directory
+	if err != fs.ErrorIsFile {
+		remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
+		wrappedFs, err = fs.NewFs(remotePath)
 	}
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remote)
+		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
 	}
 	f := &Fs{
 		Fs:     wrappedFs,
@@ -201,7 +148,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		opt:    *opt,
 		cipher: cipher,
 	}
-	cache.PinUntilFinalized(f.Fs, f)
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
@@ -211,10 +157,22 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
-		SetTier:                 true,
-		GetTier:                 true,
-		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
-	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
+	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
+
+	doChangeNotify := wrappedFs.Features().ChangeNotify
+	if doChangeNotify != nil {
+		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
+			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+				decrypted, err := f.DecryptFileName(path)
+				if err != nil {
+					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+					return
+				}
+				notifyFunc(decrypted, entryType)
+			}
+			return doChangeNotify(wrappedNotifyFunc, pollInterval)
+		}
+	}
 
 	return f, err
 }
@@ -224,22 +182,19 @@ type Options struct {
 	Remote                  string `config:"remote"`
 	FilenameEncryption      string `config:"filename_encryption"`
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
-	NoDataEncryption        bool   `config:"no_data_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
-	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
 }
 
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
-	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
 	features *fs.Features // optional features
-	cipher   *Cipher
+	cipher   Cipher
 }
 
 // Name of the remote (as passed into NewFs)
@@ -276,8 +231,8 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	*entries = append(*entries, f.newObject(obj))
 }
 
-// Encrypt a directory file name to entries.
-func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
+// Encrypt an directory file name to entries.
+func (f *Fs) addDir(entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
@@ -287,18 +242,18 @@ func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Director
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
-	*entries = append(*entries, f.newDir(ctx, dir))
+	*entries = append(*entries, f.newDir(dir))
 }
 
 // Encrypt some directory entries.  This alters entries returning it as newEntries.
-func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
+func (f *Fs) encryptEntries(entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
 	for _, entry := range entries {
 		switch x := entry.(type) {
 		case fs.Object:
 			f.add(&newEntries, x)
 		case fs.Directory:
-			f.addDir(ctx, &newEntries, x)
+			f.addDir(&newEntries, x)
 		default:
 			return nil, errors.Errorf("Unknown object type %T", entry)
 		}
@@ -315,12 +270,12 @@ func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntr
 //
 // This should return ErrDirNotFound if the directory isn't
 // found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entries, err = f.Fs.List(ctx, f.cipher.EncryptDirName(dir))
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	entries, err = f.Fs.List(f.cipher.EncryptDirName(dir))
 	if err != nil {
 		return nil, err
 	}
-	return f.encryptEntries(ctx, entries)
+	return f.encryptEntries(entries)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -339,9 +294,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // Don't implement this unless you have a more efficient way
 // of listing recursively that doing a directory traversal.
-func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	return f.Fs.Features().ListR(ctx, f.cipher.EncryptDirName(dir), func(entries fs.DirEntries) error {
-		newEntries, err := f.encryptEntries(ctx, entries)
+func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
+	return f.Fs.Features().ListR(f.cipher.EncryptDirName(dir), func(entries fs.DirEntries) error {
+		newEntries, err := f.encryptEntries(entries)
 		if err != nil {
 			return err
 		}
@@ -350,24 +305,20 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 }
 
 // NewObject finds the Object at remote.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	o, err := f.Fs.NewObject(ctx, f.cipher.EncryptFileName(remote))
+func (f *Fs) NewObject(remote string) (fs.Object, error) {
+	o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
 	return f.newObject(o), nil
 }
 
-type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
+type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
 
 // put implements Put or PutStream
-func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
-	if f.opt.NoDataEncryption {
-		return put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
-	}
-
+func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
 	// Encrypt the data into wrappedIn
-	wrappedIn, encrypter, err := f.cipher.encryptData(in)
+	wrappedIn, err := f.cipher.EncryptData(in)
 	if err != nil {
 		return nil, err
 	}
@@ -391,7 +342,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	}
 
 	// Transfer the data
-	o, err := put(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce), options...)
+	o, err := put(wrappedIn, f.newObjectInfo(src), options...)
 	if err != nil {
 		return nil, err
 	}
@@ -400,20 +351,17 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	if ht != hash.None && hasher != nil {
 		srcHash := hasher.Sums()[ht]
 		var dstHash string
-		dstHash, err = o.Hash(ctx, ht)
+		dstHash, err = o.Hash(ht)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read destination hash")
 		}
-		if srcHash != "" && dstHash != "" {
-			if srcHash != dstHash {
-				// remove object
-				err = o.Remove(ctx)
-				if err != nil {
-					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
-				}
-				return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
+		if srcHash != "" && dstHash != "" && srcHash != dstHash {
+			// remove object
+			err = o.Remove()
+			if err != nil {
+				fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 			}
-			fs.Debugf(src, "%v = %s OK", ht, srcHash)
+			return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
 		}
 	}
 
@@ -425,13 +373,13 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 // May create the object even if it returns an error - if so
 // will return the object and the error, otherwise will return
 // nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.put(ctx, in, src, options, f.Fs.Put)
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.put(in, src, options, f.Fs.Put)
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.put(ctx, in, src, options, f.Fs.Features().PutStream)
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.put(in, src, options, f.Fs.Features().PutStream)
 }
 
 // Hashes returns the supported hash sets.
@@ -442,32 +390,32 @@ func (f *Fs) Hashes() hash.Set {
 // Mkdir makes the directory (container, bucket)
 //
 // Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	return f.Fs.Mkdir(ctx, f.cipher.EncryptDirName(dir))
+func (f *Fs) Mkdir(dir string) error {
+	return f.Fs.Mkdir(f.cipher.EncryptDirName(dir))
 }
 
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.Fs.Rmdir(ctx, f.cipher.EncryptDirName(dir))
+func (f *Fs) Rmdir(dir string) error {
+	return f.Fs.Rmdir(f.cipher.EncryptDirName(dir))
 }
 
-// Purge all files in the directory specified
+// Purge all files in the root and the root directory
 //
 // Implement this if you have a way of deleting all the files
 // quicker than just running Remove() on the result of List()
 //
 // Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge() error {
 	do := f.Fs.Features().Purge
 	if do == nil {
 		return fs.ErrorCantPurge
 	}
-	return do(ctx, f.cipher.EncryptDirName(dir))
+	return do()
 }
 
-// Copy src to this remote using server-side copy operations.
+// Copy src to this remote using server side copy operations.
 //
 // This is stored with the remote path given
 //
@@ -476,7 +424,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	do := f.Fs.Features().Copy
 	if do == nil {
 		return nil, fs.ErrorCantCopy
@@ -485,14 +433,14 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if !ok {
 		return nil, fs.ErrorCantCopy
 	}
-	oResult, err := do(ctx, o.Object, f.cipher.EncryptFileName(remote))
+	oResult, err := do(o.Object, f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
 	return f.newObject(oResult), nil
 }
 
-// Move src to this remote using server-side move operations.
+// Move src to this remote using server side move operations.
 //
 // This is stored with the remote path given
 //
@@ -501,7 +449,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantMove
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	do := f.Fs.Features().Move
 	if do == nil {
 		return nil, fs.ErrorCantMove
@@ -510,7 +458,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if !ok {
 		return nil, fs.ErrorCantMove
 	}
-	oResult, err := do(ctx, o.Object, f.cipher.EncryptFileName(remote))
+	oResult, err := do(o.Object, f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
@@ -518,14 +466,14 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
+// using server side move operations.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantDirMove
 //
 // If destination exists then return fs.ErrorDirExists
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	do := f.Fs.Features().DirMove
 	if do == nil {
 		return fs.ErrorCantDirMove
@@ -535,23 +483,23 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	return do(ctx, srcFs.Fs, f.cipher.EncryptDirName(srcRemote), f.cipher.EncryptDirName(dstRemote))
+	return do(srcFs.Fs, f.cipher.EncryptDirName(srcRemote), f.cipher.EncryptDirName(dstRemote))
 }
 
 // PutUnchecked uploads the object
 //
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
-func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	do := f.Fs.Features().PutUnchecked
 	if do == nil {
 		return nil, errors.New("can't PutUnchecked")
 	}
-	wrappedIn, encrypter, err := f.cipher.encryptData(in)
+	wrappedIn, err := f.cipher.EncryptData(in)
 	if err != nil {
 		return nil, err
 	}
-	o, err := do(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce))
+	o, err := do(wrappedIn, f.newObjectInfo(src))
 	if err != nil {
 		return nil, err
 	}
@@ -562,21 +510,21 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 //
 // Implement this if you have a way of emptying the trash or
 // otherwise cleaning up old versions of files.
-func (f *Fs) CleanUp(ctx context.Context) error {
+func (f *Fs) CleanUp() error {
 	do := f.Fs.Features().CleanUp
 	if do == nil {
 		return errors.New("can't CleanUp")
 	}
-	return do(ctx)
+	return do()
 }
 
 // About gets quota information from the Fs
-func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+func (f *Fs) About() (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
 		return nil, errors.New("About not supported")
 	}
-	return do(ctx)
+	return do()
 }
 
 // UnWrap returns the Fs that this Fs is wrapping
@@ -584,16 +532,6 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
-// WrapFs returns the Fs that is wrapping this Fs
-func (f *Fs) WrapFs() fs.Fs {
-	return f.wrapper
-}
-
-// SetWrapper sets the Fs that is wrapping this Fs
-func (f *Fs) SetWrapper(wrapper fs.Fs) {
-	f.wrapper = wrapper
-}
-
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -604,53 +542,18 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 	return f.cipher.DecryptFileName(encryptedFileName)
 }
 
-// computeHashWithNonce takes the nonce and encrypts the contents of
-// src with it, and calculates the hash given by HashType on the fly
-//
-// Note that we break lots of encapsulation in this function.
-func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Object, hashType hash.Type) (hashStr string, err error) {
-	// Open the src for input
-	in, err := src.Open(ctx)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to open src")
-	}
-	defer fs.CheckClose(in, &err)
-
-	// Now encrypt the src with the nonce
-	out, err := f.cipher.newEncrypter(in, &nonce)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make encrypter")
-	}
-
-	// pipe into hash
-	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make hasher")
-	}
-	_, err = io.Copy(m, out)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to hash data")
-	}
-
-	return m.Sums()[hashType], nil
-}
-
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calculates the hash given by HashType on the fly
+// src with it, and calcuates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
-func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
-	if f.opt.NoDataEncryption {
-		return src.Hash(ctx, hashType)
-	}
-
+func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
 	// Read the nonce - opening the file is sufficient to read the nonce in
 	// use a limited read so we only read the header
-	in, err := o.Object.Open(ctx, &fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
+	in, err := o.Object.Open(&fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
 	if err != nil {
 		return "", errors.Wrap(err, "failed to open object to read nonce")
 	}
-	d, err := f.cipher.newDecrypter(in)
+	d, err := f.cipher.(*cipher).newDecrypter(in)
 	if err != nil {
 		_ = in.Close()
 		return "", errors.Wrap(err, "failed to open object to read nonce")
@@ -675,137 +578,30 @@ func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType
 		return "", errors.Wrap(err, "failed to close nonce read")
 	}
 
-	return f.computeHashWithNonce(ctx, nonce, src, hashType)
-}
-
-// MergeDirs merges the contents of all the directories passed
-// in into the first one and rmdirs the other directories.
-func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
-	do := f.Fs.Features().MergeDirs
-	if do == nil {
-		return errors.New("MergeDirs not supported")
-	}
-	out := make([]fs.Directory, len(dirs))
-	for i, dir := range dirs {
-		out[i] = fs.NewDirCopy(ctx, dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
-	}
-	return do(ctx, out)
-}
-
-// DirCacheFlush resets the directory cache - used in testing
-// as an optional interface
-func (f *Fs) DirCacheFlush() {
-	do := f.Fs.Features().DirCacheFlush
-	if do != nil {
-		do()
-	}
-}
-
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	do := f.Fs.Features().PublicLink
-	if do == nil {
-		return "", errors.New("PublicLink not supported")
-	}
-	o, err := f.NewObject(ctx, remote)
+	// Open the src for input
+	in, err = src.Open()
 	if err != nil {
-		// assume it is a directory
-		return do(ctx, f.cipher.EncryptDirName(remote), expire, unlink)
+		return "", errors.Wrap(err, "failed to open src")
 	}
-	return do(ctx, o.(*Object).Object.Remote(), expire, unlink)
-}
+	defer fs.CheckClose(in, &err)
 
-// ChangeNotify calls the passed function with a path
-// that has had changes. If the implementation
-// uses polling, it should adhere to the given interval.
-func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
-	do := f.Fs.Features().ChangeNotify
-	if do == nil {
-		return
+	// Now encrypt the src with the nonce
+	out, err := f.cipher.(*cipher).newEncrypter(in, &nonce)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make encrypter")
 	}
-	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
-		var (
-			err       error
-			decrypted string
-		)
-		switch entryType {
-		case fs.EntryDirectory:
-			decrypted, err = f.cipher.DecryptDirName(path)
-		case fs.EntryObject:
-			decrypted, err = f.cipher.DecryptFileName(path)
-		default:
-			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
-			return
-		}
-		if err != nil {
-			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-			return
-		}
-		notifyFunc(decrypted, entryType)
+
+	// pipe into hash
+	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make hasher")
 	}
-	do(ctx, wrappedNotifyFunc, pollIntervalChan)
-}
-
-var commandHelp = []fs.CommandHelp{
-	{
-		Name:  "encode",
-		Short: "Encode the given filename(s)",
-		Long: `This encodes the filenames given as arguments returning a list of
-strings of the encoded results.
-
-Usage Example:
-
-    rclone backend encode crypt: file1 [file2...]
-    rclone rc backend/command command=encode fs=crypt: file1 [file2...]
-`,
-	},
-	{
-		Name:  "decode",
-		Short: "Decode the given filename(s)",
-		Long: `This decodes the filenames given as arguments returning a list of
-strings of the decoded results. It will return an error if any of the
-inputs are invalid.
-
-Usage Example:
-
-    rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
-    rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
-`,
-	},
-}
-
-// Command the backend to run a named command
-//
-// The command run is name
-// args may be used to read arguments from
-// opts may be used to read optional arguments from
-//
-// The result should be capable of being JSON encoded
-// If it is a string or a []string it will be shown to the user
-// otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	switch name {
-	case "decode":
-		out := make([]string, 0, len(arg))
-		for _, encryptedFileName := range arg {
-			fileName, err := f.DecryptFileName(encryptedFileName)
-			if err != nil {
-				return out, errors.Wrap(err, fmt.Sprintf("Failed to decrypt : %s", encryptedFileName))
-			}
-			out = append(out, fileName)
-		}
-		return out, nil
-	case "encode":
-		out := make([]string, 0, len(arg))
-		for _, fileName := range arg {
-			encryptedFileName := f.EncryptFileName(fileName)
-			out = append(out, encryptedFileName)
-		}
-		return out, nil
-	default:
-		return nil, fs.ErrorCommandNotFound
+	_, err = io.Copy(m, out)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to hash data")
 	}
+
+	return m.Sums()[hashType], nil
 }
 
 // Object describes a wrapped for being read from the Fs
@@ -849,20 +645,16 @@ func (o *Object) Remote() string {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	size := o.Object.Size()
-	if !o.f.opt.NoDataEncryption {
-		var err error
-		size, err = o.f.cipher.DecryptedSize(size)
-		if err != nil {
-			fs.Debugf(o, "Bad size for decrypt: %v", err)
-		}
+	size, err := o.f.cipher.DecryptedSize(o.Object.Size())
+	if err != nil {
+		fs.Debugf(o, "Bad size for decrypt: %v", err)
 	}
 	return size
 }
 
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
-func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
+func (o *Object) Hash(ht hash.Type) (string, error) {
 	return "", hash.ErrUnsupported
 }
 
@@ -872,11 +664,7 @@ func (o *Object) UnWrap() fs.Object {
 }
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
-	if o.f.opt.NoDataEncryption {
-		return o.Object.Open(ctx, options...)
-	}
-
+func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
@@ -890,10 +678,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			openOptions = append(openOptions, option)
 		}
 	}
-	rc, err = o.f.cipher.DecryptDataSeek(ctx, func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
+	rc, err = o.f.cipher.DecryptDataSeek(func(underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
 		if underlyingOffset == 0 && underlyingLimit < 0 {
 			// Open with no seek
-			return o.Object.Open(ctx, openOptions...)
+			return o.Object.Open(openOptions...)
 		}
 		// Open stream with a range of underlyingOffset, underlyingLimit
 		end := int64(-1)
@@ -904,7 +692,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 		newOpenOptions := append(openOptions, &fs.RangeOption{Start: underlyingOffset, End: end})
-		return o.Object.Open(ctx, newOpenOptions...)
+		return o.Object.Open(newOpenOptions...)
 	}, offset, limit)
 	if err != nil {
 		return nil, err
@@ -913,17 +701,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 }
 
 // Update in to the object with the modTime given of the given size
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	update := func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-		return o.Object, o.Object.Update(ctx, in, src, options...)
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	update := func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+		return o.Object, o.Object.Update(in, src, options...)
 	}
-	_, err := o.f.put(ctx, in, src, options, update)
+	_, err := o.f.put(in, src, options, update)
 	return err
 }
 
 // newDir returns a dir with the Name decrypted
-func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory {
-	newDir := fs.NewDirCopy(ctx, dir)
+func (f *Fs) newDir(dir fs.Directory) fs.Directory {
+	newDir := fs.NewDirCopy(dir)
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
@@ -934,48 +722,18 @@ func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory {
 	return newDir
 }
 
-// UserInfo returns info about the connected user
-func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
-	do := f.Fs.Features().UserInfo
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	return do(ctx)
-}
-
-// Disconnect the current user
-func (f *Fs) Disconnect(ctx context.Context) error {
-	do := f.Fs.Features().Disconnect
-	if do == nil {
-		return fs.ErrorNotImplemented
-	}
-	return do(ctx)
-}
-
-// Shutdown the backend, closing any background tasks and any
-// cached connections.
-func (f *Fs) Shutdown(ctx context.Context) error {
-	do := f.Fs.Features().Shutdown
-	if do == nil {
-		return nil
-	}
-	return do(ctx)
-}
-
 // ObjectInfo describes a wrapped fs.ObjectInfo for being the source
 //
 // This encrypts the remote name and adjusts the size
 type ObjectInfo struct {
 	fs.ObjectInfo
-	f     *Fs
-	nonce nonce
+	f *Fs
 }
 
-func (f *Fs) newObjectInfo(src fs.ObjectInfo, nonce nonce) *ObjectInfo {
+func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo {
 	return &ObjectInfo{
 		ObjectInfo: src,
 		f:          f,
-		nonce:      nonce,
 	}
 }
 
@@ -1000,55 +758,10 @@ func (o *ObjectInfo) Size() int64 {
 
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
-func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
-	var srcObj fs.Object
-	var ok bool
-	// Get the underlying object if there is one
-	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
-		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
-		// Otherwise likely is an operations.OverrideRemote
-		srcObj = do.UnWrap()
-	} else {
-		return "", nil
-	}
-	// if this is wrapping a local object then we work out the hash
-	if srcObj.Fs().Features().IsLocal {
-		// Read the data and encrypt it to calculate the hash
-		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
-		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
-	}
+func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
-// ID returns the ID of the Object if known, or "" if not
-func (o *Object) ID() string {
-	do, ok := o.Object.(fs.IDer)
-	if !ok {
-		return ""
-	}
-	return do.ID()
-}
-
-// SetTier performs changing storage tier of the Object if
-// multiple storage classes supported
-func (o *Object) SetTier(tier string) error {
-	do, ok := o.Object.(fs.SetTierer)
-	if !ok {
-		return errors.New("crypt: underlying remote does not support SetTier")
-	}
-	return do.SetTier(tier)
-}
-
-// GetTier returns storage tier or class of the Object
-func (o *Object) GetTier() string {
-	do, ok := o.Object.(fs.GetTierer)
-	if !ok {
-		return ""
-	}
-	return do.GetTier()
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1056,25 +769,13 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.Commander       = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
-	_ fs.Wrapper         = (*Fs)(nil)
-	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirCacheFlusher = (*Fs)(nil)
-	_ fs.ChangeNotifier  = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
-	_ fs.UserInfoer      = (*Fs)(nil)
-	_ fs.Disconnecter    = (*Fs)(nil)
-	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
-	_ fs.IDer            = (*Object)(nil)
-	_ fs.SetTierer       = (*Object)(nil)
-	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_internal_test.go

@@ -1,143 +0,0 @@
-package crypt
-
-import (
-	"bytes"
-	"context"
-	"crypto/md5"
-	"fmt"
-	"io"
-	"testing"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type testWrapper struct {
-	fs.ObjectInfo
-}
-
-// UnWrap returns the Object that this Object is wrapping or nil if it
-// isn't wrapping anything
-func (o testWrapper) UnWrap() fs.Object {
-	if o, ok := o.ObjectInfo.(fs.Object); ok {
-		return o
-	}
-	return nil
-}
-
-// Create a temporary local fs to upload things from
-
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
-	localFs, err := fs.TemporaryLocalFs(context.Background())
-	require.NoError(t, err)
-	cleanup = func() {
-		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	}
-	return localFs, cleanup
-}
-
-// Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
-	inBuf := bytes.NewBufferString(contents)
-	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
-	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
-	obj, err := f.Put(context.Background(), inBuf, upSrc)
-	require.NoError(t, err)
-	cleanup = func() {
-		require.NoError(t, obj.Remove(context.Background()))
-	}
-	return obj, cleanup
-}
-
-// Test the ObjectInfo
-func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
-	var (
-		contents = random.String(100)
-		path     = "hash_test_object"
-		ctx      = context.Background()
-	)
-	if wrap {
-		path = "_wrap"
-	}
-
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
-
-	obj, cleanupObj := uploadFile(t, localFs, path, contents)
-	defer cleanupObj()
-
-	// encrypt the data
-	inBuf := bytes.NewBufferString(contents)
-	var outBuf bytes.Buffer
-	enc, err := f.cipher.newEncrypter(inBuf, nil)
-	require.NoError(t, err)
-	nonce := enc.nonce // read the nonce at the start
-	_, err = io.Copy(&outBuf, enc)
-	require.NoError(t, err)
-
-	var oi fs.ObjectInfo = obj
-	if wrap {
-		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = testWrapper{oi}
-	}
-
-	// wrap the object in a crypt for upload using the nonce we
-	// saved from the encrypter
-	src := f.newObjectInfo(oi, nonce)
-
-	// Test ObjectInfo methods
-	assert.Equal(t, int64(outBuf.Len()), src.Size())
-	assert.Equal(t, f, src.Fs())
-	assert.NotEqual(t, path, src.Remote())
-
-	// Test ObjectInfo.Hash
-	wantHash := md5.Sum(outBuf.Bytes())
-	gotHash, err := src.Hash(ctx, hash.MD5)
-	require.NoError(t, err)
-	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
-}
-
-func testComputeHash(t *testing.T, f *Fs) {
-	var (
-		contents = random.String(100)
-		path     = "compute_hash_test"
-		ctx      = context.Background()
-		hashType = f.Fs.Hashes().GetOne()
-	)
-
-	if hashType == hash.None {
-		t.Skipf("%v: does not support hashes", f.Fs)
-	}
-
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
-
-	// Upload a file to localFs as a test object
-	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
-	defer cleanupLocalObj()
-
-	// Upload the same data to the remote Fs also
-	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
-	defer cleanupRemoteObj()
-
-	// Calculate the expected Hash of the remote object
-	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)
-	require.NoError(t, err)
-
-	// Test computed hash matches remote object hash
-	remoteObjHash, err := remoteObj.(*Object).Object.Hash(ctx, hashType)
-	require.NoError(t, err)
-	assert.Equal(t, remoteObjHash, computedHash)
-}
-
-// InternalTest is called by fstests.Run to extra tests
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("ObjectInfo", func(t *testing.T) { testObjectInfo(t, f, false) })
-	t.Run("ObjectInfoWrap", func(t *testing.T) { testObjectInfo(t, f, true) })
-	t.Run("ComputeHash", func(t *testing.T) { testComputeHash(t, f) })
-}

backend/crypt/crypt_test.go

@@ -6,33 +6,14 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive" // for integration tests
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/swift" // for integration tests
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/crypt"
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	if *fstest.RemoteName == "" {
-		t.Skip("Skipping as -remote not set")
-	}
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   *fstest.RemoteName,
-		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-	})
-}
-
 // TestStandard runs integration tests against the remote
 func TestStandard(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
 	name := "TestCrypt"
 	fstests.Run(t, &fstests.Opt{
@@ -44,16 +25,11 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
 // TestOff runs integration tests against the remote
 func TestOff(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-off")
 	name := "TestCrypt2"
 	fstests.Run(t, &fstests.Opt{
@@ -65,16 +41,11 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
 // TestObfuscate runs integration tests against the remote
 func TestObfuscate(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt3"
 	fstests.Run(t, &fstests.Opt{
@@ -86,31 +57,6 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-	})
-}
-
-// TestNoDataObfuscate runs integration tests against the remote
-func TestNoDataObfuscate(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
-	name := "TestCrypt4"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*crypt.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "crypt"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
-			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
-			{Name: name, Key: "no_data_encryption", Value: "true"},
-		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		SkipBadWindowsCharacters: true,
 	})
 }

backend/drive/drive_internal_test.go

@@ -1,87 +1,62 @@
 package drive
 
 import (
-	"bytes"
-	"context"
 	"encoding/json"
-	"io"
-	"io/ioutil"
-	"mime"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
 	"testing"
-	"time"
+
+	"google.golang.org/api/drive/v3"
 
 	"github.com/pkg/errors"
-	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/api/drive/v3"
 )
 
-func TestDriveScopes(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		want     []string
-		wantFlag bool
-	}{
-		{"", []string{
-			"https://www.googleapis.com/auth/drive",
-		}, false},
-		{" drive.file , drive.readonly", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.readonly",
-		}, false},
-		{" drive.file , drive.appfolder", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.appfolder",
-		}, true},
-	} {
-		got := driveScopes(test.in)
-		assert.Equal(t, test.want, got, test.in)
-		gotFlag := driveScopesContainsAppFolder(got)
-		assert.Equal(t, test.wantFlag, gotFlag, test.in)
-	}
-}
-
-/*
-var additionalMimeTypes = map[string]string{
-	"application/vnd.ms-excel.sheet.macroenabled.12":                          ".xlsm",
-	"application/vnd.ms-excel.template.macroenabled.12":                       ".xltm",
-	"application/vnd.ms-powerpoint.presentation.macroenabled.12":              ".pptm",
-	"application/vnd.ms-powerpoint.slideshow.macroenabled.12":                 ".ppsm",
-	"application/vnd.ms-powerpoint.template.macroenabled.12":                  ".potm",
-	"application/vnd.ms-powerpoint":                                           ".ppt",
-	"application/vnd.ms-word.document.macroenabled.12":                        ".docm",
-	"application/vnd.ms-word.template.macroenabled.12":                        ".dotm",
-	"application/vnd.openxmlformats-officedocument.presentationml.template":   ".potx",
-	"application/vnd.openxmlformats-officedocument.spreadsheetml.template":    ".xltx",
-	"application/vnd.openxmlformats-officedocument.wordprocessingml.template": ".dotx",
-	"application/vnd.sun.xml.writer":                                          ".sxw",
-	"text/richtext":                                                           ".rtf",
-}
-*/
+const exampleExportFormats = `{
+	"application/vnd.google-apps.document": [
+		"application/rtf",
+		"application/vnd.oasis.opendocument.text",
+		"text/html",
+		"application/pdf",
+		"application/epub+zip",
+		"application/zip",
+		"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+		"text/plain"
+	],
+	"application/vnd.google-apps.spreadsheet": [
+		"application/x-vnd.oasis.opendocument.spreadsheet",
+		"text/tab-separated-values",
+		"application/pdf",
+		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+		"text/csv",
+		"application/zip",
+		"application/vnd.oasis.opendocument.spreadsheet"
+	],
+	"application/vnd.google-apps.jam": [
+		"application/pdf"
+	],
+	"application/vnd.google-apps.script": [
+		"application/vnd.google-apps.script+json"
+	],
+	"application/vnd.google-apps.presentation": [
+		"application/vnd.oasis.opendocument.presentation",
+		"application/pdf",
+		"application/vnd.openxmlformats-officedocument.presentationml.presentation",
+		"text/plain"
+	],
+	"application/vnd.google-apps.form": [
+		"application/zip"
+	],
+	"application/vnd.google-apps.drawing": [
+		"image/svg+xml",
+		"image/png",
+		"application/pdf",
+		"image/jpeg"
+	]
+}`
 
 // Load the example export formats into exportFormats for testing
-func TestInternalLoadExampleFormats(t *testing.T) {
-	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
-	var about struct {
-		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
-		ImportFormats map[string][]string `json:"importFormats,omitempty"`
-	}
-	require.NoError(t, err)
-	require.NoError(t, json.Unmarshal(buf, &about))
-	_exportFormats = fixMimeTypeMap(about.ExportFormats)
-	_importFormats = fixMimeTypeMap(about.ImportFormats)
+func TestInternalLoadExampleExportFormats(t *testing.T) {
+	exportFormatsOnce.Do(func() {})
+	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats))
 }
 
 func TestInternalParseExtensions(t *testing.T) {
@@ -90,28 +65,30 @@ func TestInternalParseExtensions(t *testing.T) {
 		want    []string
 		wantErr error
 	}{
-		{"doc", []string{".doc"}, nil},
-		{" docx ,XLSX, 	pptx,svg", []string{".docx", ".xlsx", ".pptx", ".svg"}, nil},
-		{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
-		{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
+		{"doc", []string{"doc"}, nil},
+		{" docx ,XLSX, 	pptx,svg", []string{"docx", "xlsx", "pptx", "svg"}, nil},
+		{"docx,svg,Docx", []string{"docx", "svg"}, nil},
+		{"docx,potato,docx", []string{"docx"}, errors.New(`couldn't find mime type for extension "potato"`)},
 	} {
-		extensions, _, gotErr := parseExtensions(test.in)
+		f := new(Fs)
+		gotErr := f.parseExtensions(test.in)
 		if test.wantErr == nil {
 			assert.NoError(t, gotErr)
 		} else {
 			assert.EqualError(t, gotErr, test.wantErr.Error())
 		}
-		assert.Equal(t, test.want, extensions)
+		assert.Equal(t, test.want, f.extensions)
 	}
 
 	// Test it is appending
-	extensions, _, gotErr := parseExtensions("docx,svg", "docx,svg,xlsx")
-	assert.NoError(t, gotErr)
-	assert.Equal(t, []string{".docx", ".svg", ".xlsx"}, extensions)
+	f := new(Fs)
+	assert.Nil(t, f.parseExtensions("docx,svg"))
+	assert.Nil(t, f.parseExtensions("docx,svg,xlsx"))
+	assert.Equal(t, []string{"docx", "svg", "xlsx"}, f.extensions)
+
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
-	ctx := context.Background()
 	item := &drive.File{
 		Name:     "file",
 		MimeType: "application/vnd.google-apps.document",
@@ -122,17 +99,17 @@ func TestInternalFindExportFormat(t *testing.T) {
 		wantMimeType  string
 	}{
 		{[]string{}, "", ""},
-		{[]string{".pdf"}, ".pdf", "application/pdf"},
-		{[]string{".pdf", ".rtf", ".xls"}, ".pdf", "application/pdf"},
-		{[]string{".xls", ".rtf", ".pdf"}, ".rtf", "application/rtf"},
-		{[]string{".xls", ".csv", ".svg"}, "", ""},
+		{[]string{"pdf"}, "pdf", "application/pdf"},
+		{[]string{"pdf", "rtf", "xls"}, "pdf", "application/pdf"},
+		{[]string{"xls", "rtf", "pdf"}, "rtf", "application/rtf"},
+		{[]string{"xls", "csv", "svg"}, "", ""},
 	} {
 		f := new(Fs)
-		f.exportExtensions = test.extensions
-		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
+		f.extensions = test.extensions
+		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
 		assert.Equal(t, test.wantExtension, gotExtension)
 		if test.wantExtension != "" {
-			assert.Equal(t, item.Name+gotExtension, gotFilename)
+			assert.Equal(t, item.Name+"."+gotExtension, gotFilename)
 		} else {
 			assert.Equal(t, "", gotFilename)
 		}
@@ -140,344 +117,3 @@ func TestInternalFindExportFormat(t *testing.T) {
 		assert.Equal(t, true, gotIsDocument)
 	}
 }
-
-func TestMimeTypesToExtension(t *testing.T) {
-	for mimeType, extension := range _mimeTypeToExtension {
-		extensions, err := mime.ExtensionsByType(mimeType)
-		assert.NoError(t, err)
-		assert.Contains(t, extensions, extension)
-	}
-}
-
-func TestExtensionToMimeType(t *testing.T) {
-	for mimeType, extension := range _mimeTypeToExtension {
-		gotMimeType := mime.TypeByExtension(extension)
-		mediatype, _, err := mime.ParseMediaType(gotMimeType)
-		assert.NoError(t, err)
-		assert.Equal(t, mimeType, mediatype)
-	}
-}
-
-func TestExtensionsForExportFormats(t *testing.T) {
-	if _exportFormats == nil {
-		t.Error("exportFormats == nil")
-	}
-	for fromMT, toMTs := range _exportFormats {
-		for _, toMT := range toMTs {
-			if !isInternalMimeType(toMT) {
-				extensions, err := mime.ExtensionsByType(toMT)
-				assert.NoError(t, err, "invalid MIME type %q", toMT)
-				assert.NotEmpty(t, extensions, "No extension found for %q (from: %q)", fromMT, toMT)
-			}
-		}
-	}
-}
-
-func TestExtensionsForImportFormats(t *testing.T) {
-	t.Skip()
-	if _importFormats == nil {
-		t.Error("_importFormats == nil")
-	}
-	for fromMT := range _importFormats {
-		if !isInternalMimeType(fromMT) {
-			extensions, err := mime.ExtensionsByType(fromMT)
-			assert.NoError(t, err, "invalid MIME type %q", fromMT)
-			assert.NotEmpty(t, extensions, "No extension found for %q", fromMT)
-		}
-	}
-}
-
-func (f *Fs) InternalTestDocumentImport(t *testing.T) {
-	oldAllow := f.opt.AllowImportNameChange
-	f.opt.AllowImportNameChange = true
-	defer func() {
-		f.opt.AllowImportNameChange = oldAllow
-	}()
-
-	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
-	require.NoError(t, err)
-
-	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
-	require.NoError(t, err)
-
-	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
-	require.NoError(t, err)
-
-	err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.doc", "example2.doc")
-	require.NoError(t, err)
-}
-
-func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
-	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
-	require.NoError(t, err)
-
-	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
-	require.NoError(t, err)
-
-	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
-	require.NoError(t, err)
-
-	err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.xlsx", "example1.ods")
-	require.NoError(t, err)
-}
-
-func (f *Fs) InternalTestDocumentExport(t *testing.T) {
-	var buf bytes.Buffer
-	var err error
-
-	f.exportExtensions, _, err = parseExtensions("txt")
-	require.NoError(t, err)
-
-	obj, err := f.NewObject(context.Background(), "example2.txt")
-	require.NoError(t, err)
-
-	rc, err := obj.Open(context.Background())
-	require.NoError(t, err)
-	defer func() { require.NoError(t, rc.Close()) }()
-
-	_, err = io.Copy(&buf, rc)
-	require.NoError(t, err)
-	text := buf.String()
-
-	for _, excerpt := range []string{
-		"Lorem ipsum dolor sit amet, consectetur",
-		"porta at ultrices in, consectetur at augue.",
-	} {
-		require.Contains(t, text, excerpt)
-	}
-}
-
-func (f *Fs) InternalTestDocumentLink(t *testing.T) {
-	var buf bytes.Buffer
-	var err error
-
-	f.exportExtensions, _, err = parseExtensions("link.html")
-	require.NoError(t, err)
-
-	obj, err := f.NewObject(context.Background(), "example2.link.html")
-	require.NoError(t, err)
-
-	rc, err := obj.Open(context.Background())
-	require.NoError(t, err)
-	defer func() { require.NoError(t, rc.Close()) }()
-
-	_, err = io.Copy(&buf, rc)
-	require.NoError(t, err)
-	text := buf.String()
-
-	require.True(t, strings.HasPrefix(text, "<html>"))
-	require.True(t, strings.HasSuffix(text, "</html>\n"))
-	for _, excerpt := range []string{
-		`<meta http-equiv="refresh"`,
-		`Loading <a href="`,
-	} {
-		require.Contains(t, text, excerpt)
-	}
-}
-
-const (
-	// from fstest/fstests/fstests.go
-	existingDir    = "hello? sausage"
-	existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
-	existingSubDir = "êé"
-)
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
-func (f *Fs) InternalTestShortcuts(t *testing.T) {
-	ctx := context.Background()
-	srcObj, err := f.NewObject(ctx, existingFile)
-	require.NoError(t, err)
-	srcHash, err := srcObj.Hash(ctx, hash.MD5)
-	require.NoError(t, err)
-	assert.NotEqual(t, "", srcHash)
-	t.Run("Errors", func(t *testing.T) {
-		_, err := f.makeShortcut(ctx, "", f, "")
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "can't be root")
-
-		_, err = f.makeShortcut(ctx, "notfound", f, "dst")
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "can't find source")
-
-		_, err = f.makeShortcut(ctx, existingFile, f, existingFile)
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "not overwriting")
-		assert.Contains(t, err.Error(), "existing file")
-
-		_, err = f.makeShortcut(ctx, existingFile, f, existingDir)
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "not overwriting")
-		assert.Contains(t, err.Error(), "existing directory")
-	})
-	t.Run("File", func(t *testing.T) {
-		dstObj, err := f.makeShortcut(ctx, existingFile, f, "shortcut.txt")
-		require.NoError(t, err)
-		require.NotNil(t, dstObj)
-		assert.Equal(t, "shortcut.txt", dstObj.Remote())
-		dstHash, err := dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-	})
-	t.Run("Dir", func(t *testing.T) {
-		dstObj, err := f.makeShortcut(ctx, existingDir, f, "shortcutdir")
-		require.NoError(t, err)
-		require.Nil(t, dstObj)
-		entries, err := f.List(ctx, "shortcutdir")
-		require.NoError(t, err)
-		require.Equal(t, 1, len(entries))
-		require.Equal(t, "shortcutdir/"+existingSubDir, entries[0].Remote())
-		require.NoError(t, f.Rmdir(ctx, "shortcutdir"))
-	})
-	t.Run("Command", func(t *testing.T) {
-		_, err := f.Command(ctx, "shortcut", []string{"one"}, nil)
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "need exactly 2 arguments")
-
-		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
-			"target": "doesnotexistremote:",
-		})
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "couldn't find target")
-
-		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
-			"target": ".",
-		})
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "target is not a drive backend")
-
-		dstObjI, err := f.Command(ctx, "shortcut", []string{existingFile, "shortcut2.txt"}, map[string]string{
-			"target": fs.ConfigString(f),
-		})
-		require.NoError(t, err)
-		dstObj := dstObjI.(*Object)
-		assert.Equal(t, "shortcut2.txt", dstObj.Remote())
-		dstHash, err := dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-
-		dstObjI, err = f.Command(ctx, "shortcut", []string{existingFile, "shortcut3.txt"}, nil)
-		require.NoError(t, err)
-		dstObj = dstObjI.(*Object)
-		assert.Equal(t, "shortcut3.txt", dstObj.Remote())
-		dstHash, err = dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-	})
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/UnTrash
-func (f *Fs) InternalTestUnTrash(t *testing.T) {
-	ctx := context.Background()
-
-	// Make some objects, one in a subdir
-	contents := random.String(100)
-	file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
-	_, obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
-	file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
-	_, _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
-
-	// Check objects
-	checkObjects := func() {
-		fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{
-			file1,
-			file2,
-		}, []string{
-			"trashDir/subdir",
-		}, f.Precision())
-	}
-	checkObjects()
-
-	// Make sure we are using the trash
-	require.Equal(t, true, f.opt.UseTrash)
-
-	// Remove the object and the dir
-	require.NoError(t, obj1.Remove(ctx))
-	require.NoError(t, f.Purge(ctx, "trashDir/subdir"))
-
-	// Check objects gone
-	fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{}, []string{}, f.Precision())
-
-	// Restore the object and directory
-	r, err := f.unTrashDir(ctx, "trashDir", true)
-	require.NoError(t, err)
-	assert.Equal(t, unTrashResult{Errors: 0, Untrashed: 2}, r)
-
-	// Check objects restored
-	checkObjects()
-
-	// Remove the test dir
-	require.NoError(t, f.Purge(ctx, "trashDir"))
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
-	ctx := context.Background()
-	obj, err := f.NewObject(ctx, existingFile)
-	require.NoError(t, err)
-	o := obj.(*Object)
-
-	dir, err := ioutil.TempDir("", "rclone-drive-copyid-test")
-	require.NoError(t, err)
-	defer func() {
-		_ = os.RemoveAll(dir)
-	}()
-
-	checkFile := func(name string) {
-		filePath := filepath.Join(dir, name)
-		fi, err := os.Stat(filePath)
-		require.NoError(t, err)
-		assert.Equal(t, int64(100), fi.Size())
-		err = os.Remove(filePath)
-		require.NoError(t, err)
-	}
-
-	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "couldn't find id")
-	})
-
-	t.Run("Directory", func(t *testing.T) {
-		rootID, err := f.dirCache.RootID(ctx, false)
-		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
-	})
-
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
-		require.NoError(t, err)
-		checkFile(path.Base(existingFile))
-	})
-
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
-		require.NoError(t, err)
-		checkFile("potato.txt")
-	})
-}
-
-func (f *Fs) InternalTest(t *testing.T) {
-	// These tests all depend on each other so run them as nested tests
-	t.Run("DocumentImport", func(t *testing.T) {
-		f.InternalTestDocumentImport(t)
-		t.Run("DocumentUpdate", func(t *testing.T) {
-			f.InternalTestDocumentUpdate(t)
-			t.Run("DocumentExport", func(t *testing.T) {
-				f.InternalTestDocumentExport(t)
-				t.Run("DocumentLink", func(t *testing.T) {
-					f.InternalTestDocumentLink(t)
-				})
-			})
-		})
-	})
-	t.Run("Shortcuts", f.InternalTestShortcuts)
-	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/drive/drive_test.go

@@ -1,35 +1,17 @@
 // Test Drive filesystem interface
-
-package drive
+package drive_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestDrive:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize:  minChunkSize,
-			CeilChunkSize: fstests.NextPowerOfTwo,
-		},
+		NilObject:  (*drive.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/drive/test/about.json

@@ -1,178 +0,0 @@
-{
- "importFormats": {
-  "text/tab-separated-values": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/x-vnd.oasis.opendocument.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/jpeg": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/bmp": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/gif": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-excel.sheet.macroenabled.12": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.template": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-powerpoint.presentation.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-word.template.macroenabled.12": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
-   "application/vnd.google-apps.document"
-  ],
-  "image/pjpeg": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.google-apps.script+text/plain": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/vnd.ms-excel": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.sun.xml.writer": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-word.document.macroenabled.12": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.ms-powerpoint.slideshow.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "text/rtf": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/plain": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.oasis.opendocument.spreadsheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/x-vnd.oasis.opendocument.spreadsheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "image/png": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/x-vnd.oasis.opendocument.text": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/msword": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/pdf": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/json": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/x-msmetafile": [
-   "application/vnd.google-apps.drawing"
-  ],
-  "application/vnd.openxmlformats-officedocument.spreadsheetml.template": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.ms-powerpoint": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-excel.template.macroenabled.12": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "image/x-bmp": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/rtf": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.template": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/x-png": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/html": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.oasis.opendocument.text": [
-   "application/vnd.google-apps.document"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.google-apps.script+json": [
-   "application/vnd.google-apps.script"
-  ],
-  "application/vnd.openxmlformats-officedocument.presentationml.slideshow": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "application/vnd.ms-powerpoint.template.macroenabled.12": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "text/csv": [
-   "application/vnd.google-apps.spreadsheet"
-  ],
-  "application/vnd.oasis.opendocument.presentation": [
-   "application/vnd.google-apps.presentation"
-  ],
-  "image/jpg": [
-   "application/vnd.google-apps.document"
-  ],
-  "text/richtext": [
-   "application/vnd.google-apps.document"
-  ]
- },
- "exportFormats": {
-  "application/vnd.google-apps.document": [
-   "application/rtf",
-   "application/vnd.oasis.opendocument.text",
-   "text/html",
-   "application/pdf",
-   "application/epub+zip",
-   "application/zip",
-   "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-   "text/plain"
-  ],
-  "application/vnd.google-apps.spreadsheet": [
-   "application/x-vnd.oasis.opendocument.spreadsheet",
-   "text/tab-separated-values",
-   "application/pdf",
-   "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-   "text/csv",
-   "application/zip",
-   "application/vnd.oasis.opendocument.spreadsheet"
-  ],
-  "application/vnd.google-apps.jam": [
-   "application/pdf"
-  ],
-  "application/vnd.google-apps.script": [
-   "application/vnd.google-apps.script+json"
-  ],
-  "application/vnd.google-apps.presentation": [
-   "application/vnd.oasis.opendocument.presentation",
-   "application/pdf",
-   "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-   "text/plain"
-  ],
-  "application/vnd.google-apps.form": [
-   "application/zip"
-  ],
-  "application/vnd.google-apps.drawing": [
-   "image/svg+xml",
-   "image/png",
-   "application/pdf",
-   "image/jpeg"
-  ]
- }
-}

backend/drive/upload.go

@@ -11,18 +11,18 @@
 package drive
 
 import (
-	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/readers"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/lib/readers"
+	"github.com/pkg/errors"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
 )
@@ -50,13 +50,14 @@ type resumableUpload struct {
 }
 
 // Upload the io.Reader in of size bytes with contentType and info
-func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
-	params := url.Values{
-		"alt":        {"json"},
-		"uploadType": {"resumable"},
-		"fields":     {partialFields},
+func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string, info *drive.File, remote string) (*drive.File, error) {
+	params := make(url.Values)
+	params.Set("alt", "json")
+	params.Set("uploadType", "resumable")
+	params.Set("fields", partialFields)
+	if f.isTeamDrive {
+		params.Set("supportsTeamDrives", "true")
 	}
-	params.Set("supportsAllDrives", "true")
 	if f.opt.KeepRevisionForever {
 		params.Set("keepRevisionForever", "true")
 	}
@@ -77,7 +78,7 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 			return false, err
 		}
 		var req *http.Request
-		req, err = http.NewRequestWithContext(ctx, method, urls, body)
+		req, err = http.NewRequest(method, urls, body)
 		if err != nil {
 			return false, err
 		}
@@ -86,15 +87,13 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 		})
 		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
 		req.Header.Set("X-Upload-Content-Type", contentType)
-		if size >= 0 {
-			req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
-		}
+		req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
 		res, err = f.client.Do(req)
 		if err == nil {
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return f.shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -108,30 +107,60 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 		MediaType:     contentType,
 		ContentLength: size,
 	}
-	return rx.Upload(ctx)
+	return rx.Upload()
 }
 
 // Make an http.Request for the range passed in
-func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
-	req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
+func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request {
+	req, _ := http.NewRequest("POST", rx.URI, body)
 	req.ContentLength = reqSize
-	totalSize := "*"
-	if rx.ContentLength >= 0 {
-		totalSize = strconv.FormatInt(rx.ContentLength, 10)
-	}
 	if reqSize != 0 {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
 	} else {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
 	}
 	req.Header.Set("Content-Type", rx.MediaType)
 	return req
 }
 
+// rangeRE matches the transfer status response from the server. $1 is
+// the last byte index uploaded.
+var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
+
+// Query drive for the amount transferred so far
+//
+// If error is nil, then start should be valid
+func (rx *resumableUpload) transferStatus() (start int64, err error) {
+	req := rx.makeRequest(0, nil, 0)
+	res, err := rx.f.client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer googleapi.CloseBody(res)
+	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
+		return rx.ContentLength, nil
+	}
+	if res.StatusCode != statusResumeIncomplete {
+		err = googleapi.CheckResponse(res)
+		if err != nil {
+			return 0, err
+		}
+		return 0, errors.Errorf("unexpected http return code %v", res.StatusCode)
+	}
+	Range := res.Header.Get("Range")
+	if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 {
+		start, err = strconv.ParseInt(m[1], 10, 64)
+		if err == nil {
+			return start, nil
+		}
+	}
+	return 0, errors.Errorf("unable to parse range %q", Range)
+}
+
 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
-func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
+func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
 	_, _ = chunk.Seek(0, io.SeekStart)
-	req := rx.makeRequest(ctx, start, chunk, chunkSize)
+	req := rx.makeRequest(start, chunk, chunkSize)
 	res, err := rx.f.client.Do(req)
 	if err != nil {
 		return 599, err
@@ -153,7 +182,7 @@ func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionally on
+	// any other 2xx codes, so we parse it unconditionaly on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err
@@ -164,45 +193,23 @@ func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk
 
 // Upload uploads the chunks from the input
 // It retries each chunk using the pacer and --low-level-retries
-func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
+func (rx *resumableUpload) Upload() (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int
 	var err error
 	buf := make([]byte, int(rx.f.opt.ChunkSize))
-	for finished := false; !finished; {
-		var reqSize int64
-		var chunk io.ReadSeeker
-		if rx.ContentLength >= 0 {
-			// If size known use repeatable reader for smoother bwlimit
-			if start >= rx.ContentLength {
-				break
-			}
-			reqSize = rx.ContentLength - start
-			if reqSize >= int64(rx.f.opt.ChunkSize) {
-				reqSize = int64(rx.f.opt.ChunkSize)
-			}
-			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
-		} else {
-			// If size unknown read into buffer
-			var n int
-			n, err = readers.ReadFill(rx.Media, buf)
-			if err == io.EOF {
-				// Send the last chunk with the correct ContentLength
-				// otherwise Google doesn't know we've finished
-				rx.ContentLength = start + int64(n)
-				finished = true
-			} else if err != nil {
-				return nil, err
-			}
-			reqSize = int64(n)
-			chunk = bytes.NewReader(buf[:reqSize])
+	for start < rx.ContentLength {
+		reqSize := rx.ContentLength - start
+		if reqSize >= int64(rx.f.opt.ChunkSize) {
+			reqSize = int64(rx.f.opt.ChunkSize)
 		}
+		chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 
 		// Transfer the chunk
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
-			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
-			again, err := rx.f.shouldRetry(ctx, err)
+			StatusCode, err = rx.transferChunk(start, chunk, reqSize)
+			again, err := shouldRetry(err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/dbhash/dbhash_test.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/rclone/rclone/backend/dropbox/dbhash"
+	"github.com/ncw/rclone/backend/dropbox/dbhash"
 	"github.com/stretchr/testify/assert"
 )
 

backend/dropbox/dropbox_internal_test.go

@@ -1,44 +0,0 @@
-package dropbox
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestInternalCheckPathLength(t *testing.T) {
-	rep := func(n int, r rune) (out string) {
-		rs := make([]rune, n)
-		for i := range rs {
-			rs[i] = r
-		}
-		return string(rs)
-	}
-	for _, test := range []struct {
-		in string
-		ok bool
-	}{
-		{in: "", ok: true},
-		{in: rep(maxFileNameLength, 'a'), ok: true},
-		{in: rep(maxFileNameLength+1, 'a'), ok: false},
-		{in: rep(maxFileNameLength, '£'), ok: true},
-		{in: rep(maxFileNameLength+1, '£'), ok: false},
-		{in: rep(maxFileNameLength, '☺'), ok: true},
-		{in: rep(maxFileNameLength+1, '☺'), ok: false},
-		{in: rep(maxFileNameLength, '你'), ok: true},
-		{in: rep(maxFileNameLength+1, '你'), ok: false},
-		{in: "/ok/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength, 'a') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, 'a') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '£') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '£') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '☺') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '☺') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '你') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '你') + "/ok", ok: false},
-	} {
-
-		err := checkPathLength(test.in)
-		assert.Equal(t, test.ok, err == nil, test.in)
-	}
-}

backend/dropbox/dropbox_test.go

@@ -1,26 +1,17 @@
 // Test Dropbox filesystem interface
-package dropbox
+package dropbox_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/dropbox"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestDropbox:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MaxChunkSize: maxChunkSize,
-		},
+		NilObject:  (*dropbox.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/fichier/api.go

@@ -1,496 +0,0 @@
-package fichier
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	403, // Forbidden (may happen when request limit is exceeded)
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	// Detect this error which the integration tests provoke
-	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
-	//
-	// https://1fichier.com/api.html
-	//
-	// file/ls.cgi is limited :
-	//
-	// Warning (can be changed in case of abuses) :
-	// List all files of the account is limited to 1 request per hour.
-	// List folders is limited to 5 000 results and 1 request per folder per 30s.
-	if err != nil && strings.Contains(err.Error(), "Flood detected") {
-		fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
-		time.Sleep(30 * time.Second)
-	}
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
-
-func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) {
-	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
-	if err != nil {
-		return
-	}
-	// Temporary Object under construction
-	o = &Object{
-		fs:     f,
-		remote: remote,
-	}
-	return o, leaf, directoryID, nil
-}
-
-func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
-	request := FileInfoRequest{
-		URL: url,
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/info.cgi",
-	}
-
-	var file File
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't read file info")
-	}
-
-	return &file, err
-}
-
-func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
-	request := DownloadRequest{
-		URL:    url,
-		Single: 1,
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/download/get_token.cgi",
-	}
-
-	var token GetTokenResponse
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't list files")
-	}
-
-	return &token, nil
-}
-
-func fileFromSharedFile(file *SharedFile) File {
-	return File{
-		URL:      file.Link,
-		Filename: file.Filename,
-		Size:     file.Size,
-	}
-}
-
-func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) {
-	opts := rest.Opts{
-		Method:     "GET",
-		RootURL:    "https://1fichier.com/dir/",
-		Path:       id,
-		Parameters: map[string][]string{"json": {"1"}},
-	}
-
-	var sharedFiles SharedFolderResponse
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't list files")
-	}
-
-	entries = make([]fs.DirEntry, len(sharedFiles))
-
-	for i, sharedFile := range sharedFiles {
-		entries[i] = f.newObjectFromFile(ctx, "", fileFromSharedFile(&sharedFile))
-	}
-
-	return entries, nil
-}
-
-func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) {
-	// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
-	request := ListFilesRequest{
-		FolderID: directoryID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/ls.cgi",
-	}
-
-	filesList = &FilesList{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't list files")
-	}
-	for i := range filesList.Items {
-		item := &filesList.Items[i]
-		item.Filename = f.opt.Enc.ToStandardName(item.Filename)
-	}
-
-	return filesList, nil
-}
-
-func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) {
-	// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
-
-	request := ListFolderRequest{
-		FolderID: directoryID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/ls.cgi",
-	}
-
-	foldersList = &FoldersList{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't list folders")
-	}
-	foldersList.Name = f.opt.Enc.ToStandardName(foldersList.Name)
-	for i := range foldersList.SubFolders {
-		folder := &foldersList.SubFolders[i]
-		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
-	}
-
-	// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
-
-	return foldersList, err
-}
-
-func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-
-	files, err := f.listFiles(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	folders, err := f.listFolders(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
-
-	for i, item := range files.Items {
-		entries[i] = f.newObjectFromFile(ctx, dir, item)
-	}
-
-	for i, folder := range folders.SubFolders {
-		createDate, err := time.Parse("2006-01-02 15:04:05", folder.CreateDate)
-		if err != nil {
-			return nil, err
-		}
-
-		fullPath := getRemote(dir, folder.Name)
-		folderID := strconv.Itoa(folder.ID)
-
-		entries[len(files.Items)+i] = fs.NewDir(fullPath, createDate).SetID(folderID)
-
-		// fs.Debugf(f, "Put Path `%s` for id `%d` into dircache", fullPath, folder.ID)
-		f.dirCache.Put(fullPath, folderID)
-	}
-
-	return entries, nil
-}
-
-func (f *Fs) newObjectFromFile(ctx context.Context, dir string, item File) *Object {
-	return &Object{
-		fs:     f,
-		remote: getRemote(dir, item.Filename),
-		file:   item,
-	}
-}
-
-func getRemote(dir, fileName string) string {
-	if dir == "" {
-		return fileName
-	}
-
-	return dir + "/" + fileName
-}
-
-func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
-	name := f.opt.Enc.FromStandardName(leaf)
-	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
-
-	request := MakeFolderRequest{
-		FolderID: folderID,
-		Name:     name,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/mkdir.cgi",
-	}
-
-	response = &MakeFolderResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't create folder")
-	}
-
-	// fs.Debugf(f, "Created Folder `%s` in id `%s`", name, directoryID)
-
-	return response, err
-}
-
-func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) {
-	// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
-
-	request := &RemoveFolderRequest{
-		FolderID: folderID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/rm.cgi",
-	}
-
-	response = &GenericOKResponse{}
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't remove folder")
-	}
-	if response.Status != "OK" {
-		return nil, errors.New("Can't remove non-empty dir")
-	}
-
-	// fs.Debugf(f, "Removed Folder with id `%s`", directoryID)
-
-	return response, nil
-}
-
-func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) {
-	request := &RemoveFileRequest{
-		Files: []RmFile{
-			{url},
-		},
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/rm.cgi",
-	}
-
-	response = &GenericOKResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't remove file")
-	}
-
-	// fs.Debugf(f, "Removed file with url `%s`", url)
-
-	return response, nil
-}
-
-func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename string) (response *MoveFileResponse, err error) {
-	request := &MoveFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/mv.cgi",
-	}
-
-	response = &MoveFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't copy file")
-	}
-
-	return response, nil
-}
-
-func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
-	request := &CopyFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/cp.cgi",
-	}
-
-	response = &CopyFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't copy file")
-	}
-
-	return response, nil
-}
-
-func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
-	// fs.Debugf(f, "Requesting Upload node")
-
-	opts := rest.Opts{
-		Method:      "GET",
-		ContentType: "application/json", // 1Fichier API is bad
-		Path:        "/upload/get_upload_server.cgi",
-	}
-
-	response = &GetUploadNodeResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "didnt got an upload node")
-	}
-
-	// fs.Debugf(f, "Got Upload node")
-
-	return response, err
-}
-
-func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string, options ...fs.OpenOption) (response *http.Response, err error) {
-	// fs.Debugf(f, "Uploading File `%s`", fileName)
-
-	fileName = f.opt.Enc.FromStandardName(fileName)
-
-	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/upload.cgi",
-		Parameters: map[string][]string{
-			"id": {uploadID},
-		},
-		NoResponse:           true,
-		Body:                 in,
-		ContentLength:        &size,
-		Options:              options,
-		MultipartContentName: "file[]",
-		MultipartFileName:    fileName,
-		MultipartParams: map[string][]string{
-			"did": {folderID},
-		},
-	}
-
-	if node != "" {
-		opts.RootURL = "https://" + node
-	}
-
-	err = f.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't upload file")
-	}
-
-	// fs.Debugf(f, "Uploaded File `%s`", fileName)
-
-	return response, err
-}
-
-func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
-	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
-
-	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("Invalid UploadID")
-	}
-
-	opts := rest.Opts{
-		Method:  "GET",
-		Path:    "/end.pl",
-		RootURL: "https://" + nodeurl,
-		Parameters: map[string][]string{
-			"xid": {uploadID},
-		},
-		ExtraHeaders: map[string]string{
-			"JSON": "1",
-		},
-	}
-
-	response = &EndFileUploadResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't finish file upload")
-	}
-
-	return response, err
-}

backend/fichier/fichier.go

@@ -1,503 +0,0 @@
-package fichier
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/dircache"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	rootID         = "0"
-	apiBaseURL     = "https://api.1fichier.com/v1"
-	minSleep       = 400 * time.Millisecond // api is extremely rate limited now
-	maxSleep       = 5 * time.Second
-	decayConstant  = 2 // bigger for slower decay, exponential
-	attackConstant = 0 // start with max sleep
-)
-
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "fichier",
-		Description: "1Fichier",
-		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
-			Name: "api_key",
-		}, {
-			Help:     "If you want to download a shared folder, add this parameter",
-			Name:     "shared_folder",
-			Required: false,
-			Advanced: true,
-		}, {
-			Name:     config.ConfigEncoding,
-			Help:     config.ConfigEncodingHelp,
-			Advanced: true,
-			// Characters that need escaping
-			//
-			// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-			// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-			// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-			// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-			// 		'\'': '＇', // FULLWIDTH APOSTROPHE
-			// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
-			// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
-			//
-			// Leading space and trailing space
-			Default: (encoder.Display |
-				encoder.EncodeBackSlash |
-				encoder.EncodeSingleQuote |
-				encoder.EncodeBackQuote |
-				encoder.EncodeDoubleQuote |
-				encoder.EncodeLtGt |
-				encoder.EncodeDollar |
-				encoder.EncodeLeftSpace |
-				encoder.EncodeRightSpace |
-				encoder.EncodeInvalidUtf8),
-		}},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	APIKey       string               `config:"api_key"`
-	SharedFolder string               `config:"shared_folder"`
-	Enc          encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs is the interface a cloud storage system must provide
-type Fs struct {
-	root       string
-	name       string
-	features   *fs.Features
-	opt        Options
-	dirCache   *dircache.DirCache
-	baseClient *http.Client
-	pacer      *fs.Pacer
-	rest       *rest.Client
-}
-
-// FindLeaf finds a directory of name leaf in the folder with ID pathID
-func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
-	folderID, err := strconv.Atoi(pathID)
-	if err != nil {
-		return "", false, err
-	}
-	folders, err := f.listFolders(ctx, folderID)
-	if err != nil {
-		return "", false, err
-	}
-
-	for _, folder := range folders.SubFolders {
-		if folder.Name == leaf {
-			pathIDOut := strconv.Itoa(folder.ID)
-			return pathIDOut, true, nil
-		}
-	}
-
-	return "", false, nil
-}
-
-// CreateDir makes a directory with pathID as parent and name leaf
-func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
-	folderID, err := strconv.Atoi(pathID)
-	if err != nil {
-		return "", err
-	}
-	resp, err := f.makeFolder(ctx, leaf, folderID)
-	if err != nil {
-		return "", err
-	}
-	return strconv.Itoa(resp.FolderID), err
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a description of the FS
-func (f *Fs) String() string {
-	return fmt.Sprintf("1Fichier root '%s'", f.root)
-}
-
-// Precision of the ModTimes in this Fs
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash types of the filesystem
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.Whirlpool)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// NewFs makes a new Fs object from the path
-//
-// The path is of the form remote:path
-//
-// Remotes are looked up in the config file.  If the remote isn't
-// found then NotFoundInConfigFile will be returned.
-//
-// On Windows avoid single character remote names as they can be mixed
-// up with drive letters.
-func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(config, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// If using a Shared Folder override root
-	if opt.SharedFolder != "" {
-		root = ""
-	}
-
-	//workaround for wonky parser
-	root = strings.Trim(root, "/")
-
-	f := &Fs{
-		name:       name,
-		root:       root,
-		opt:        *opt,
-		pacer:      fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
-		baseClient: &http.Client{},
-	}
-
-	f.features = (&fs.Features{
-		DuplicateFiles:          true,
-		CanHaveEmptyDirectories: true,
-		ReadMimeType:            true,
-	}).Fill(ctx, f)
-
-	client := fshttp.NewClient(ctx)
-
-	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
-
-	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
-
-	f.dirCache = dircache.New(root, rootID, f)
-
-	// Find the current root
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		// Assume it is a file
-		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
-		tempF.root = newRoot
-		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(ctx, false)
-		if err != nil {
-			// No root so return old f
-			return f, nil
-		}
-		_, err := tempF.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				// File doesn't exist so return old f
-				return f, nil
-			}
-			return nil, err
-		}
-		f.features.Fill(ctx, &tempF)
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/rclone/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if f.opt.SharedFolder != "" {
-		return f.listSharedFiles(ctx, f.opt.SharedFolder)
-	}
-
-	dirContent, err := f.listDir(ctx, dir)
-	if err != nil {
-		return nil, err
-	}
-
-	return dirContent, nil
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		if err == fs.ErrorDirNotFound {
-			return nil, fs.ErrorObjectNotFound
-		}
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	files, err := f.listFiles(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files.Items {
-		if file.Filename == leaf {
-			path, ok := f.dirCache.GetInv(directoryID)
-
-			if !ok {
-				return nil, errors.New("Cannot find dir in dircache")
-			}
-
-			return f.newObjectFromFile(ctx, path, file), nil
-		}
-	}
-
-	return nil, fs.ErrorObjectNotFound
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
-// return an error or upload it properly (rather than e.g. calling panic).
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.NewObject(ctx, src.Remote())
-	switch err {
-	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
-	case fs.ErrorObjectNotFound:
-		// Not found so create it
-		return f.PutUnchecked(ctx, in, src, options...)
-	default:
-		return nil, err
-	}
-}
-
-// putUnchecked uploads the object with the given name and size
-//
-// This will create a duplicate if we upload a new file without
-// checking to see if there is one already - use Put() for that.
-func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
-	if size > int64(300e9) {
-		return nil, errors.New("File too big, cant upload")
-	} else if size == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	nodeResponse, err := f.getUploadNode(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
-	if err != nil {
-		return nil, err
-	}
-
-	fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(fileUploadResponse.Links) == 0 {
-		return nil, errors.New("upload response not found")
-	} else if len(fileUploadResponse.Links) > 1 {
-		fs.Debugf(remote, "Multiple upload responses found, using the first")
-	}
-
-	link := fileUploadResponse.Links[0]
-	fileSize, err := strconv.ParseInt(link.Size, 10, 64)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &Object{
-		fs:     f,
-		remote: remote,
-		file: File{
-			CDN:         0,
-			Checksum:    link.Whirlpool,
-			ContentType: "",
-			Date:        time.Now().Format("2006-01-02 15:04:05"),
-			Filename:    link.Filename,
-			Pass:        0,
-			Size:        fileSize,
-			URL:         link.Download,
-		},
-	}, nil
-}
-
-// PutUnchecked uploads the object
-//
-// This will create a duplicate if we upload a new file without
-// checking to see if there is one already - use Put() for that.
-func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...)
-}
-
-// Mkdir makes the directory (container, bucket)
-//
-// Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	_, err := f.dirCache.FindDir(ctx, dir, true)
-	return err
-}
-
-// Rmdir removes the directory (container, bucket) if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return err
-	}
-
-	_, err = f.removeFolder(ctx, dir, folderID)
-	if err != nil {
-		return err
-	}
-
-	f.dirCache.FlushDir(dir)
-
-	return nil
-}
-
-// Move src to this remote using server side move operations.
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't move file")
-	}
-	if resp.Status != "OK" {
-		return nil, errors.New("couldn't move file")
-	}
-
-	file, err := f.readFileInfo(ctx, resp.URLs[0])
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// Copy src to this remote using server side move operations.
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf)
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't move file")
-	}
-	if resp.Status != "OK" {
-		return nil, errors.New("couldn't move file")
-	}
-
-	file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL)
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	o, err := f.NewObject(ctx, remote)
-	if err != nil {
-		return "", err
-	}
-	return o.(*Object).file.URL, nil
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs              = (*Fs)(nil)
-	_ fs.Mover           = (*Fs)(nil)
-	_ fs.Copier          = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
-	_ fs.PutUncheckeder  = (*Fs)(nil)
-	_ dircache.DirCacher = (*Fs)(nil)
-)

backend/fichier/fichier_test.go

@@ -1,15 +0,0 @@
-// Test 1Fichier filesystem interface
-package fichier
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFichier:",
-	})
-}

backend/fichier/object.go

@@ -1,162 +0,0 @@
-package fichier
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// Object is a filesystem like object provided by an Fs
-type Object struct {
-	fs     *Fs
-	remote string
-	file   File
-}
-
-// String returns a description of the Object
-func (o *Object) String() string {
-	return o.file.Filename
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification date of the file
-// It should return a best guess if one isn't available
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	modTime, err := time.Parse("2006-01-02 15:04:05", o.file.Date)
-
-	if err != nil {
-		return time.Now()
-	}
-
-	return modTime
-}
-
-// Size returns the size of the file
-func (o *Object) Size() int64 {
-	return o.file.Size
-}
-
-// Fs returns read only access to the Fs that this object is part of
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns the selected checksum of the file
-// If no checksum is available it returns ""
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.Whirlpool {
-		return "", hash.ErrUnsupported
-	}
-
-	return o.file.Checksum, nil
-}
-
-// Storable says whether this object can be stored
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the metadata on the object to set the modification date
-func (o *Object) SetModTime(context.Context, time.Time) error {
-	return fs.ErrorCantSetModTime
-	//return errors.New("setting modtime is not supported for 1fichier remotes")
-}
-
-func (o *Object) setMetaData(file File) {
-	o.file = file
-}
-
-// Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	fs.FixRangeOption(options, o.file.Size)
-	downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: downloadToken.URL,
-		Options: options,
-	}
-
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.rest.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, err
-	}
-	return resp.Body, err
-}
-
-// Update in to the object with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
-// return an error or update the object properly (rather than e.g. calling panic).
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if src.Size() < 0 {
-		return errors.New("refusing to update with unknown size")
-	}
-
-	// upload with new size but old name
-	info, err := o.fs.putUnchecked(ctx, in, o.Remote(), src.Size(), options...)
-	if err != nil {
-		return err
-	}
-
-	// Delete duplicate after successful upload
-	err = o.Remove(ctx)
-	if err != nil {
-		return errors.Wrap(err, "failed to remove old version")
-	}
-
-	// Replace guts of old object with new one
-	*o = *info.(*Object)
-
-	return nil
-}
-
-// Remove removes this object
-func (o *Object) Remove(ctx context.Context) error {
-	// fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL)
-
-	_, err := o.fs.deleteFile(ctx, o.file.URL)
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType(ctx context.Context) string {
-	return o.file.ContentType
-}
-
-// ID returns the ID of the Object if known, or "" if not
-func (o *Object) ID() string {
-	return o.file.URL
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Object    = (*Object)(nil)
-	_ fs.MimeTyper = (*Object)(nil)
-	_ fs.IDer      = (*Object)(nil)
-)

backend/fichier/structs.go

@@ -1,157 +0,0 @@
-package fichier
-
-// FileInfoRequest is the request structure of the corresponding request
-type FileInfoRequest struct {
-	URL string `json:"url"`
-}
-
-// ListFolderRequest is the request structure of the corresponding request
-type ListFolderRequest struct {
-	FolderID int `json:"folder_id"`
-}
-
-// ListFilesRequest is the request structure of the corresponding request
-type ListFilesRequest struct {
-	FolderID int `json:"folder_id"`
-}
-
-// DownloadRequest is the request structure of the corresponding request
-type DownloadRequest struct {
-	URL    string `json:"url"`
-	Single int    `json:"single"`
-}
-
-// RemoveFolderRequest is the request structure of the corresponding request
-type RemoveFolderRequest struct {
-	FolderID int `json:"folder_id"`
-}
-
-// RemoveFileRequest is the request structure of the corresponding request
-type RemoveFileRequest struct {
-	Files []RmFile `json:"files"`
-}
-
-// RmFile is the request structure of the corresponding request
-type RmFile struct {
-	URL string `json:"url"`
-}
-
-// GenericOKResponse is the response structure of the corresponding request
-type GenericOKResponse struct {
-	Status  string `json:"status"`
-	Message string `json:"message"`
-}
-
-// MakeFolderRequest is the request structure of the corresponding request
-type MakeFolderRequest struct {
-	Name     string `json:"name"`
-	FolderID int    `json:"folder_id"`
-}
-
-// MakeFolderResponse is the response structure of the corresponding request
-type MakeFolderResponse struct {
-	Name     string `json:"name"`
-	FolderID int    `json:"folder_id"`
-}
-
-// MoveFileRequest is the request structure of the corresponding request
-type MoveFileRequest struct {
-	URLs     []string `json:"urls"`
-	FolderID int      `json:"destination_folder_id"`
-	Rename   string   `json:"rename,omitempty"`
-}
-
-// MoveFileResponse is the response structure of the corresponding request
-type MoveFileResponse struct {
-	Status string   `json:"status"`
-	URLs   []string `json:"urls"`
-}
-
-// CopyFileRequest is the request structure of the corresponding request
-type CopyFileRequest struct {
-	URLs     []string `json:"urls"`
-	FolderID int      `json:"folder_id"`
-	Rename   string   `json:"rename,omitempty"`
-}
-
-// CopyFileResponse is the response structure of the corresponding request
-type CopyFileResponse struct {
-	Status string     `json:"status"`
-	Copied int        `json:"copied"`
-	URLs   []FileCopy `json:"urls"`
-}
-
-// FileCopy is used in the the CopyFileResponse
-type FileCopy struct {
-	FromURL string `json:"from_url"`
-	ToURL   string `json:"to_url"`
-}
-
-// GetUploadNodeResponse is the response structure of the corresponding request
-type GetUploadNodeResponse struct {
-	ID  string `json:"id"`
-	URL string `json:"url"`
-}
-
-// GetTokenResponse is the response structure of the corresponding request
-type GetTokenResponse struct {
-	URL     string `json:"url"`
-	Status  string `json:"Status"`
-	Message string `json:"Message"`
-}
-
-// SharedFolderResponse is the response structure of the corresponding request
-type SharedFolderResponse []SharedFile
-
-// SharedFile is the structure how 1Fichier returns a shared File
-type SharedFile struct {
-	Filename string `json:"filename"`
-	Link     string `json:"link"`
-	Size     int64  `json:"size"`
-}
-
-// EndFileUploadResponse is the response structure of the corresponding request
-type EndFileUploadResponse struct {
-	Incoming int `json:"incoming"`
-	Links    []struct {
-		Download  string `json:"download"`
-		Filename  string `json:"filename"`
-		Remove    string `json:"remove"`
-		Size      string `json:"size"`
-		Whirlpool string `json:"whirlpool"`
-	} `json:"links"`
-}
-
-// File is the structure how 1Fichier returns a File
-type File struct {
-	CDN         int    `json:"cdn"`
-	Checksum    string `json:"checksum"`
-	ContentType string `json:"content-type"`
-	Date        string `json:"date"`
-	Filename    string `json:"filename"`
-	Pass        int    `json:"pass"`
-	Size        int64  `json:"size"`
-	URL         string `json:"url"`
-}
-
-// FilesList is the structure how 1Fichier returns a list of files
-type FilesList struct {
-	Items  []File `json:"items"`
-	Status string `json:"Status"`
-}
-
-// Folder is the structure how 1Fichier returns a Folder
-type Folder struct {
-	CreateDate string `json:"create_date"`
-	ID         int    `json:"id"`
-	Name       string `json:"name"`
-	Pass       int    `json:"pass"`
-}
-
-// FoldersList is the structure how 1Fichier returns a list of Folders
-type FoldersList struct {
-	FolderID   int      `json:"folder_id"`
-	Name       string   `json:"name"`
-	Status     string   `json:"Status"`
-	SubFolders []Folder `json:"sub_folders"`
-}

backend/filefabric/api/types.go

@@ -1,391 +0,0 @@
-// Package api has type definitions for filefabric
-//
-// Converted from the API responses with help from https://mholt.github.io/json-to-go/
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"reflect"
-	"strings"
-	"time"
-)
-
-const (
-	// TimeFormat for parameters (UTC)
-	timeFormatParameters = `2006-01-02 15:04:05`
-	// "2020-08-11 10:10:04" for JSON parsing
-	timeFormatJSON = `"` + timeFormatParameters + `"`
-)
-
-// Time represents represents date and time information for the
-// filefabric API
-type Time time.Time
-
-// MarshalJSON turns a Time into JSON (in UTC)
-func (t *Time) MarshalJSON() (out []byte, err error) {
-	timeString := (*time.Time)(t).UTC().Format(timeFormatJSON)
-	return []byte(timeString), nil
-}
-
-var zeroTime = []byte(`"0000-00-00 00:00:00"`)
-
-// UnmarshalJSON turns JSON into a Time (in UTC)
-func (t *Time) UnmarshalJSON(data []byte) error {
-	// Set a Zero time.Time if we receive a zero time input
-	if bytes.Equal(data, zeroTime) {
-		*t = Time(time.Time{})
-		return nil
-	}
-	newT, err := time.Parse(timeFormatJSON, string(data))
-	if err != nil {
-		return err
-	}
-	*t = Time(newT)
-	return nil
-}
-
-// String turns a Time into a string in UTC suitable for the API
-// parameters
-func (t Time) String() string {
-	return time.Time(t).UTC().Format(timeFormatParameters)
-}
-
-// Status return returned in all status responses
-type Status struct {
-	Code    string `json:"status"`
-	Message string `json:"statusmessage"`
-	TaskID  string `json:"taskid"`
-	// Warning string `json:"warning"` // obsolete
-}
-
-// Status statisfies the error interface
-func (e *Status) Error() string {
-	return fmt.Sprintf("%s (%s)", e.Message, e.Code)
-}
-
-// OK returns true if the status is all good
-func (e *Status) OK() bool {
-	return e.Code == "ok"
-}
-
-// GetCode returns the status code if any
-func (e *Status) GetCode() string {
-	return e.Code
-}
-
-// OKError defines an interface for items which can be OK or be an error
-type OKError interface {
-	error
-	OK() bool
-	GetCode() string
-}
-
-// Check Status satisfies the OKError interface
-var _ OKError = (*Status)(nil)
-
-// EmptyResponse is response which just returns the error condition
-type EmptyResponse struct {
-	Status
-}
-
-// GetTokenByAuthTokenResponse is the response to getTokenByAuthToken
-type GetTokenByAuthTokenResponse struct {
-	Status
-	Token              string `json:"token"`
-	UserID             string `json:"userid"`
-	AllowLoginRemember string `json:"allowloginremember"`
-	LastLogin          Time   `json:"lastlogin"`
-	AutoLoginCode      string `json:"autologincode"`
-}
-
-// ApplianceInfo is the response to getApplianceInfo
-type ApplianceInfo struct {
-	Status
-	Sitetitle            string `json:"sitetitle"`
-	OauthLoginSupport    string `json:"oauthloginsupport"`
-	IsAppliance          string `json:"isappliance"`
-	SoftwareVersion      string `json:"softwareversion"`
-	SoftwareVersionLabel string `json:"softwareversionlabel"`
-}
-
-// GetFolderContentsResponse is returned from getFolderContents
-type GetFolderContentsResponse struct {
-	Status
-	Total  int    `json:"total,string"`
-	Items  []Item `json:"filelist"`
-	Folder Item   `json:"folder"`
-	From   int    `json:"from,string"`
-	//Count         int    `json:"count"`
-	Pid           string `json:"pid"`
-	RefreshResult Status `json:"refreshresult"`
-	// Curfolder         Item              `json:"curfolder"` - sometimes returned as "ROOT"?
-	Parents           []Item            `json:"parents"`
-	CustomPermissions CustomPermissions `json:"custompermissions"`
-}
-
-// ItemType determine whether it is a file or a folder
-type ItemType uint8
-
-// Types of things in Item
-const (
-	ItemTypeFile   ItemType = 0
-	ItemTypeFolder ItemType = 1
-)
-
-// Item ia a File or a Folder
-type Item struct {
-	ID  string `json:"fi_id"`
-	PID string `json:"fi_pid"`
-	// UID             string   `json:"fi_uid"`
-	Name string `json:"fi_name"`
-	// S3Name          string   `json:"fi_s3name"`
-	// Extension       string   `json:"fi_extension"`
-	// Description     string   `json:"fi_description"`
-	Type ItemType `json:"fi_type,string"`
-	// Created         Time     `json:"fi_created"`
-	Size        int64  `json:"fi_size,string"`
-	ContentType string `json:"fi_contenttype"`
-	// Tags            string   `json:"fi_tags"`
-	// MainCode        string   `json:"fi_maincode"`
-	// Public          int      `json:"fi_public,string"`
-	// Provider        string   `json:"fi_provider"`
-	// ProviderFolder  string   `json:"fi_providerfolder"` // folder
-	// Encrypted       int      `json:"fi_encrypted,string"`
-	// StructType      string   `json:"fi_structtype"`
-	// Bname           string   `json:"fi_bname"` // folder
-	// OrgID           string   `json:"fi_orgid"`
-	// Favorite        int      `json:"fi_favorite,string"`
-	// IspartOf        string   `json:"fi_ispartof"` // folder
-	Modified Time `json:"fi_modified"`
-	// LastAccessed    Time     `json:"fi_lastaccessed"`
-	// Hits            int64    `json:"fi_hits,string"`
-	// IP              string   `json:"fi_ip"` // folder
-	// BigDescription  string   `json:"fi_bigdescription"`
-	LocalTime Time `json:"fi_localtime"`
-	// OrgfolderID     string   `json:"fi_orgfolderid"`
-	// StorageIP       string   `json:"fi_storageip"` // folder
-	// RemoteTime      Time     `json:"fi_remotetime"`
-	// ProviderOptions string   `json:"fi_provideroptions"`
-	// Access          string   `json:"fi_access"`
-	// Hidden          string   `json:"fi_hidden"` // folder
-	// VersionOf       string   `json:"fi_versionof"`
-	Trash bool `json:"trash"`
-	// Isbucket        string   `json:"isbucket"` // filelist
-	SubFolders int64 `json:"subfolders"` // folder
-}
-
-// ItemFields is a | separated list of fields in Item
-var ItemFields = mustFields(Item{})
-
-// fields returns the JSON fields in use by opt as a | separated
-// string.
-func fields(opt interface{}) (pipeTags string, err error) {
-	var tags []string
-	def := reflect.ValueOf(opt)
-	defType := def.Type()
-	for i := 0; i < def.NumField(); i++ {
-		field := defType.Field(i)
-		tag, ok := field.Tag.Lookup("json")
-		if !ok {
-			continue
-		}
-		if comma := strings.IndexRune(tag, ','); comma >= 0 {
-			tag = tag[:comma]
-		}
-		if tag == "" {
-			continue
-		}
-		tags = append(tags, tag)
-	}
-	return strings.Join(tags, "|"), nil
-}
-
-// mustFields returns the JSON fields in use by opt as a | separated
-// string. It panics on failure.
-func mustFields(opt interface{}) string {
-	tags, err := fields(opt)
-	if err != nil {
-		panic(err)
-	}
-	return tags
-}
-
-// CustomPermissions is returned as part of GetFolderContentsResponse
-type CustomPermissions struct {
-	Upload            string `json:"upload"`
-	CreateSubFolder   string `json:"createsubfolder"`
-	Rename            string `json:"rename"`
-	Delete            string `json:"delete"`
-	Move              string `json:"move"`
-	ManagePermissions string `json:"managepermissions"`
-	ListOnly          string `json:"listonly"`
-	VisibleInTrash    string `json:"visibleintrash"`
-}
-
-// DoCreateNewFolderResponse is response from foCreateNewFolder
-type DoCreateNewFolderResponse struct {
-	Status
-	Item Item `json:"file"`
-}
-
-// DoInitUploadResponse is response from doInitUpload
-type DoInitUploadResponse struct {
-	Status
-	ProviderID          string `json:"providerid"`
-	UploadCode          string `json:"uploadcode"`
-	FileType            string `json:"filetype"`
-	DirectUploadSupport string `json:"directuploadsupport"`
-	ResumeAllowed       string `json:"resumeallowed"`
-}
-
-// UploaderResponse is returned from /cgi-bin/uploader/uploader1.cgi
-//
-// Sometimes the response is returned as XML and sometimes as JSON
-type UploaderResponse struct {
-	FileSize int64  `xml:"filesize" json:"filesize,string"`
-	MD5      string `xml:"md5" json:"md5"`
-	Success  string `xml:"success" json:"success"`
-}
-
-// UploadStatus is returned from getUploadStatus
-type UploadStatus struct {
-	Status
-	UploadCode     string `json:"uploadcode"`
-	Metafile       string `json:"metafile"`
-	Percent        int    `json:"percent,string"`
-	Uploaded       int64  `json:"uploaded,string"`
-	Size           int64  `json:"size,string"`
-	Filename       string `json:"filename"`
-	Nofile         string `json:"nofile"`
-	Completed      string `json:"completed"`
-	Completsuccess string `json:"completsuccess"`
-	Completerror   string `json:"completerror"`
-}
-
-// DoCompleteUploadResponse is the response to doCompleteUpload
-type DoCompleteUploadResponse struct {
-	Status
-	UploadedSize int64  `json:"uploadedsize,string"`
-	StorageIP    string `json:"storageip"`
-	UploadedName string `json:"uploadedname"`
-	// Versioned    []interface{} `json:"versioned"`
-	// VersionedID  int           `json:"versionedid"`
-	// Comment      interface{}           `json:"comment"`
-	File Item `json:"file"`
-	// UsSize       string        `json:"us_size"`
-	// PaSize       string        `json:"pa_size"`
-	// SpaceInfo    SpaceInfo     `json:"spaceinfo"`
-}
-
-// Providers is returned as part of UploadResponse
-type Providers struct {
-	Max     string `json:"max"`
-	Used    string `json:"used"`
-	ID      string `json:"id"`
-	Private string `json:"private"`
-	Limit   string `json:"limit"`
-	Percent int    `json:"percent"`
-}
-
-// Total is returned as part of UploadResponse
-type Total struct {
-	Max        string `json:"max"`
-	Used       string `json:"used"`
-	ID         string `json:"id"`
-	Priused    string `json:"priused"`
-	Primax     string `json:"primax"`
-	Limit      string `json:"limit"`
-	Percent    int    `json:"percent"`
-	Pripercent int    `json:"pripercent"`
-}
-
-// UploadResponse is returned as part of SpaceInfo
-type UploadResponse struct {
-	Providers []Providers `json:"providers"`
-	Total     Total       `json:"total"`
-}
-
-// SpaceInfo is returned as part of DoCompleteUploadResponse
-type SpaceInfo struct {
-	Response UploadResponse `json:"response"`
-	Status   string         `json:"status"`
-}
-
-// DeleteResponse is returned from doDeleteFile
-type DeleteResponse struct {
-	Status
-	Deleted        []string      `json:"deleted"`
-	Errors         []interface{} `json:"errors"`
-	ID             string        `json:"fi_id"`
-	BackgroundTask int           `json:"backgroundtask"`
-	UsSize         string        `json:"us_size"`
-	PaSize         string        `json:"pa_size"`
-	//SpaceInfo      SpaceInfo     `json:"spaceinfo"`
-}
-
-// FileResponse is returned from doRenameFile
-type FileResponse struct {
-	Status
-	Item   Item   `json:"file"`
-	Exists string `json:"exists"`
-}
-
-// MoveFilesResponse is returned from doMoveFiles
-type MoveFilesResponse struct {
-	Status
-	Filesleft         string   `json:"filesleft"`
-	Addedtobackground string   `json:"addedtobackground"`
-	Moved             string   `json:"moved"`
-	Item              Item     `json:"file"`
-	IDs               []string `json:"fi_ids"`
-	Length            int      `json:"length"`
-	DirID             string   `json:"dir_id"`
-	MovedObjects      []Item   `json:"movedobjects"`
-	// FolderTasks       []interface{}  `json:"foldertasks"`
-}
-
-// TasksResponse is the response to getUserBackgroundTasks
-type TasksResponse struct {
-	Status
-	Tasks []Task `json:"tasks"`
-	Total string `json:"total"`
-}
-
-// BtData is part of TasksResponse
-type BtData struct {
-	Callback string `json:"callback"`
-}
-
-// Task describes a task returned in TasksResponse
-type Task struct {
-	BtID             string `json:"bt_id"`
-	UsID             string `json:"us_id"`
-	BtType           string `json:"bt_type"`
-	BtData           BtData `json:"bt_data"`
-	BtStatustext     string `json:"bt_statustext"`
-	BtStatusdata     string `json:"bt_statusdata"`
-	BtMessage        string `json:"bt_message"`
-	BtProcent        string `json:"bt_procent"`
-	BtAdded          string `json:"bt_added"`
-	BtStatus         string `json:"bt_status"`
-	BtCompleted      string `json:"bt_completed"`
-	BtTitle          string `json:"bt_title"`
-	BtCredentials    string `json:"bt_credentials"`
-	BtHidden         string `json:"bt_hidden"`
-	BtAutoremove     string `json:"bt_autoremove"`
-	BtDevsite        string `json:"bt_devsite"`
-	BtPriority       string `json:"bt_priority"`
-	BtReport         string `json:"bt_report"`
-	BtSitemarker     string `json:"bt_sitemarker"`
-	BtExecuteafter   string `json:"bt_executeafter"`
-	BtCompletestatus string `json:"bt_completestatus"`
-	BtSubtype        string `json:"bt_subtype"`
-	BtCanceled       string `json:"bt_canceled"`
-	Callback         string `json:"callback"`
-	CanBeCanceled    bool   `json:"canbecanceled"`
-	CanBeRestarted   bool   `json:"canberestarted"`
-	Type             string `json:"type"`
-	Status           string `json:"status"`
-	Settings         string `json:"settings"`
-}

backend/filefabric/filefabric_test.go

@@ -1,17 +0,0 @@
-// Test filefabric filesystem interface
-package filefabric_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/filefabric"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFileFabric:",
-		NilObject:  (*filefabric.Object)(nil),
-	})
-}

backend/ftp/ftp.go

@@ -2,42 +2,21 @@
 package ftp
 
 import (
-	"context"
-	"crypto/tls"
 	"io"
-	"net"
 	"net/textproto"
+	"os"
 	"path"
-	"runtime"
-	"strings"
 	"sync"
 	"time"
 
 	"github.com/jlaffaye/ftp"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/env"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/readers"
-)
-
-var (
-	currentUser = env.CurrentUser()
-)
-
-const (
-	minSleep      = 10 * time.Millisecond
-	maxSleep      = 2 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
 )
 
 // Register with Fs
@@ -46,125 +25,51 @@ func init() {
 		Name:        "ftp",
 		Description: "FTP Connection",
 		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Name:     "host",
-			Help:     "FTP host to connect to",
-			Required: true,
-			Examples: []fs.OptionExample{{
-				Value: "ftp.example.com",
-				Help:  "Connect to ftp.example.com",
-			}},
-		}, {
-			Name: "user",
-			Help: "FTP username, leave blank for current username, " + currentUser,
-		}, {
-			Name: "port",
-			Help: "FTP port, leave blank to use default (21)",
-		}, {
-			Name:       "pass",
-			Help:       "FTP password",
-			IsPassword: true,
-			Required:   true,
-		}, {
-			Name: "tls",
-			Help: `Use Implicit FTPS (FTP over TLS)
-When using implicit FTP over TLS the client connects using TLS
-right from the start which breaks compatibility with
-non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTP.`,
-			Default: false,
-		}, {
-			Name: "explicit_tls",
-			Help: `Use Explicit FTPS (FTP over TLS)
-When using explicit FTP over TLS the client explicitly requests
-security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTP.`,
-			Default: false,
-		}, {
-			Name:     "concurrency",
-			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-			Default:  0,
-			Advanced: true,
-		}, {
-			Name:     "no_check_certificate",
-			Help:     "Do not verify the TLS certificate of the server",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     "disable_epsv",
-			Help:     "Disable using EPSV even if server advertises support",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     "disable_mlsd",
-			Help:     "Disable using MLSD even if server advertises support",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:    "idle_timeout",
-			Default: fs.Duration(60 * time.Second),
-			Help: `Max time before closing idle connections
-
-If no connections have been returned to the connection pool in the time
-given, rclone will empty the connection pool.
-
-Set to 0 to keep connections indefinitely.
-`,
-			Advanced: true,
-		}, {
-			Name:     "close_timeout",
-			Help:     "Maximum time to wait for a response to close.",
-			Default:  fs.Duration(60 * time.Second),
-			Advanced: true,
-		}, {
-			Name:     config.ConfigEncoding,
-			Help:     config.ConfigEncodingHelp,
-			Advanced: true,
-			// The FTP protocol can't handle trailing spaces (for instance
-			// pureftpd turns them into _)
-			//
-			// proftpd can't handle '*' in file names
-			// pureftpd can't handle '[', ']' or '*'
-			Default: (encoder.Display |
-				encoder.EncodeRightSpace),
-		}},
+		Options: []fs.Option{
+			{
+				Name:     "host",
+				Help:     "FTP host to connect to",
+				Required: true,
+				Examples: []fs.OptionExample{{
+					Value: "ftp.example.com",
+					Help:  "Connect to ftp.example.com",
+				}},
+			}, {
+				Name: "user",
+				Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
+			}, {
+				Name: "port",
+				Help: "FTP port, leave blank to use default (21)",
+			}, {
+				Name:       "pass",
+				Help:       "FTP password",
+				IsPassword: true,
+				Required:   true,
+			},
+		},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string               `config:"host"`
-	User              string               `config:"user"`
-	Pass              string               `config:"pass"`
-	Port              string               `config:"port"`
-	TLS               bool                 `config:"tls"`
-	ExplicitTLS       bool                 `config:"explicit_tls"`
-	Concurrency       int                  `config:"concurrency"`
-	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
-	DisableEPSV       bool                 `config:"disable_epsv"`
-	DisableMLSD       bool                 `config:"disable_mlsd"`
-	IdleTimeout       fs.Duration          `config:"idle_timeout"`
-	CloseTimeout      fs.Duration          `config:"close_timeout"`
-	Enc               encoder.MultiEncoder `config:"encoding"`
+	Host string `config:"host"`
+	User string `config:"user"`
+	Pass string `config:"pass"`
+	Port string `config:"port"`
 }
 
 // Fs represents a remote FTP server
 type Fs struct {
-	name     string         // name of this remote
-	root     string         // the path we are working on if any
-	opt      Options        // parsed options
-	ci       *fs.ConfigInfo // global config
-	features *fs.Features   // optional features
+	name     string       // name of this remote
+	root     string       // the path we are working on if any
+	opt      Options      // parsed options
+	features *fs.Features // optional features
 	url      string
 	user     string
 	pass     string
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
-	drain    *time.Timer // used to drain the pool when we stop using the connections
-	tokens   *pacer.TokenDispenser
-	tlsConf  *tls.Config
-	pacer    *fs.Pacer // pacer for FTP connections
 }
 
 // Object describes an FTP file
@@ -204,119 +109,25 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Enable debugging output
-type debugLog struct {
-	mu   sync.Mutex
-	auth bool
-}
-
-// Write writes len(p) bytes from p to the underlying data stream. It returns
-// the number of bytes written from p (0 <= n <= len(p)) and any error
-// encountered that caused the write to stop early. Write must return a non-nil
-// error if it returns n < len(p). Write must not modify the slice data, even
-// temporarily.
-//
-// Implementations must not retain p.
-//
-// This writes debug info to the log
-func (dl *debugLog) Write(p []byte) (n int, err error) {
-	dl.mu.Lock()
-	defer dl.mu.Unlock()
-	_, file, _, ok := runtime.Caller(1)
-	direction := "FTP Rx"
-	if ok && strings.Contains(file, "multi") {
-		direction = "FTP Tx"
-	}
-	lines := strings.Split(string(p), "\r\n")
-	if lines[len(lines)-1] == "" {
-		lines = lines[:len(lines)-1]
-	}
-	for _, line := range lines {
-		if !dl.auth && strings.HasPrefix(line, "PASS") {
-			fs.Debugf(direction, "PASS *****")
-			continue
-		}
-		fs.Debugf(direction, "%q", line)
-	}
-	return len(p), nil
-}
-
-// shouldRetry returns a boolean as to whether this err deserve to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	switch errX := err.(type) {
-	case *textproto.Error:
-		switch errX.Code {
-		case ftp.StatusNotAvailable:
-			return true, err
-		}
-	}
-	return fserrors.ShouldRetry(err), err
-}
-
 // Open a new connection to the FTP server.
-func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
+func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
-
-	// Make ftp library dial with fshttp dialer optionally using TLS
-	dial := func(network, address string) (conn net.Conn, err error) {
-		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
-		}
-		return
-	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
-
-	if f.opt.TLS {
-		// Our dialer takes care of TLS but ftp library also needs tlsConf
-		// as a trigger for sending PSBZ and PROT options to server.
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
-	} else if f.opt.ExplicitTLS {
-		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
-	}
-	if f.opt.DisableEPSV {
-		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
-	}
-	if f.opt.DisableMLSD {
-		ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
-	}
-	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
-		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		c, err = ftp.Dial(f.dialAddr, ftpConfig...)
-		if err != nil {
-			return shouldRetry(ctx, err)
-		}
-		err = c.Login(f.user, f.pass)
-		if err != nil {
-			_ = c.Quit()
-			return shouldRetry(ctx, err)
-		}
-		return false, nil
-	})
+	c, err := ftp.DialTimeout(f.dialAddr, fs.Config.ConnectTimeout)
 	if err != nil {
-		err = errors.Wrapf(err, "failed to make FTP connection to %q", f.dialAddr)
+		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
+		return nil, errors.Wrap(err, "ftpConnection Dial")
 	}
-	return c, err
+	err = c.Login(f.user, f.pass)
+	if err != nil {
+		_ = c.Quit()
+		fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
+		return nil, errors.Wrap(err, "ftpConnection Login")
+	}
+	return c, nil
 }
 
 // Get an FTP connection from the pool, or open a new one
-func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		f.tokens.Get()
-	}
-	accounting.LimitTPS(ctx)
+func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -326,11 +137,7 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
 	if c != nil {
 		return c, nil
 	}
-	c, err = f.ftpConnection(ctx)
-	if err != nil && f.opt.Concurrency > 0 {
-		f.tokens.Put()
-	}
-	return c, err
+	return f.ftpConnection()
 }
 
 // Return an FTP connection to the pool
@@ -340,16 +147,7 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
-	if f.opt.Concurrency > 0 {
-		defer f.tokens.Put()
-	}
-	if pc == nil {
-		return
-	}
 	c := *pc
-	if c == nil {
-		return
-	}
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
@@ -365,34 +163,11 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
-	if f.opt.IdleTimeout > 0 {
-		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
-	}
 	f.poolMu.Unlock()
 }
 
-// Drain the pool of any connections
-func (f *Fs) drainPool(ctx context.Context) (err error) {
-	f.poolMu.Lock()
-	defer f.poolMu.Unlock()
-	if f.opt.IdleTimeout > 0 {
-		f.drain.Stop()
-	}
-	if len(f.pool) != 0 {
-		fs.Debugf(f, "closing %d unused connections", len(f.pool))
-	}
-	for i, c := range f.pool {
-		if cErr := c.Quit(); cErr != nil {
-			err = cErr
-		}
-		f.pool[i] = nil
-	}
-	f.pool = nil
-	return err
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
 	opt := new(Options)
@@ -406,7 +181,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	}
 	user := opt.User
 	if user == "" {
-		user = currentUser
+		user = os.Getenv("USER")
 	}
 	port := opt.Port
 	if port == "" {
@@ -414,44 +189,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	}
 
 	dialAddr := opt.Host + ":" + port
-	protocol := "ftp://"
-	if opt.TLS {
-		protocol = "ftps://"
-	}
-	if opt.TLS && opt.ExplicitTLS {
-		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
-	}
-	var tlsConfig *tls.Config
-	if opt.TLS || opt.ExplicitTLS {
-		tlsConfig = &tls.Config{
-			ServerName:         opt.Host,
-			InsecureSkipVerify: opt.SkipVerifyTLSCert,
-		}
-	}
-	u := protocol + path.Join(dialAddr+"/", root)
-	ci := fs.GetConfig(ctx)
+	u := "ftp://" + path.Join(dialAddr+"/", root)
 	f := &Fs{
 		name:     name,
 		root:     root,
 		opt:      *opt,
-		ci:       ci,
 		url:      u,
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
-		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
-		tlsConf:  tlsConfig,
-		pacer:    fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
-	}).Fill(ctx, f)
-	// set the pool drainer timer going
-	if f.opt.IdleTimeout > 0 {
-		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
-	}
+	}).Fill(f)
 	// Make a connection and pool it to return errors early
-	c, err := f.getFtpConnection(ctx)
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "NewFs")
 	}
@@ -463,7 +215,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		if f.root == "." {
 			f.root = ""
 		}
-		_, err := f.NewObject(ctx, remote)
+		_, err := f.NewObject(remote)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
 				// File doesn't exist so return old f
@@ -478,12 +230,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	return f, err
 }
 
-// Shutdown the backend, closing any background tasks and any
-// cached connections.
-func (f *Fs) Shutdown(ctx context.Context) error {
-	return f.drainPool(ctx)
-}
-
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
 	switch errX := err.(type) {
@@ -508,51 +254,23 @@ func translateErrorDir(err error) error {
 	return err
 }
 
-// entryToStandard converts an incoming ftp.Entry to Standard encoding
-func (f *Fs) entryToStandard(entry *ftp.Entry) {
-	// Skip . and .. as we don't want these encoded
-	if entry.Name == "." || entry.Name == ".." {
-		return
-	}
-	entry.Name = f.opt.Enc.ToStandardName(entry.Name)
-	entry.Target = f.opt.Enc.ToStandardPath(entry.Target)
-}
-
-// dirFromStandardPath returns dir in encoded form.
-func (f *Fs) dirFromStandardPath(dir string) string {
-	// Skip . and .. as we don't want these encoded
-	if dir == "." || dir == ".." {
-		return dir
-	}
-	return f.opt.Enc.FromStandardPath(dir)
-}
-
 // findItem finds a directory entry for the name in its parent directory
-func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
+func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
 	fullPath := path.Join(f.root, remote)
-	if fullPath == "" || fullPath == "." || fullPath == "/" {
-		// if root, assume exists and synthesize an entry
-		return &ftp.Entry{
-			Name: "",
-			Type: ftp.EntryTypeFolder,
-			Time: time.Now(),
-		}, nil
-	}
 	dir := path.Dir(fullPath)
 	base := path.Base(fullPath)
 
-	c, err := f.getFtpConnection(ctx)
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "findItem")
 	}
-	files, err := c.List(f.dirFromStandardPath(dir))
+	files, err := c.List(dir)
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 	for _, file := range files {
-		f.entryToStandard(file)
 		if file.Name == base {
 			return file, nil
 		}
@@ -562,9 +280,9 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
+func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(remote)
 	if err != nil {
 		return nil, err
 	}
@@ -586,8 +304,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 }
 
 // dirExists checks the directory pointed to by remote exists or not
-func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, remote)
+func (f *Fs) dirExists(remote string) (exists bool, err error) {
+	entry, err := f.findItem(remote)
 	if err != nil {
 		return false, errors.Wrap(err, "dirExists")
 	}
@@ -606,47 +324,22 @@ func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err err
 //
 // This should return ErrDirNotFound if the directory isn't
 // found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	// defer log.Trace(dir, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
-	c, err := f.getFtpConnection(ctx)
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
 	}
-
-	var listErr error
-	var files []*ftp.Entry
-
-	resultchan := make(chan []*ftp.Entry, 1)
-	errchan := make(chan error, 1)
-	go func() {
-		result, err := c.List(f.dirFromStandardPath(path.Join(f.root, dir)))
-		f.putFtpConnection(&c, err)
-		if err != nil {
-			errchan <- err
-			return
-		}
-		resultchan <- result
-	}()
-
-	// Wait for List for up to Timeout seconds
-	timer := time.NewTimer(f.ci.TimeoutOrInfinite())
-	select {
-	case listErr = <-errchan:
-		timer.Stop()
-		return nil, translateErrorDir(listErr)
-	case files = <-resultchan:
-		timer.Stop()
-	case <-timer.C:
-		// if timer fired assume no error but connection dead
-		fs.Errorf(f, "Timeout when waiting for List")
-		return nil, errors.New("Timeout when waiting for List")
+	files, err := c.List(path.Join(f.root, dir))
+	f.putFtpConnection(&c, err)
+	if err != nil {
+		return nil, translateErrorDir(err)
 	}
-
 	// Annoyingly FTP returns success for a directory which
 	// doesn't exist, so check it really doesn't exist if no
 	// entries found.
 	if len(files) == 0 {
-		exists, err := f.dirExists(ctx, dir)
+		exists, err := f.dirExists(dir)
 		if err != nil {
 			return nil, errors.Wrap(err, "list")
 		}
@@ -656,7 +349,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	for i := range files {
 		object := files[i]
-		f.entryToStandard(object)
 		newremote := path.Join(dir, object.Name)
 		switch object.Type {
 		case ftp.EntryTypeFolder:
@@ -697,9 +389,9 @@ func (f *Fs) Precision() time.Duration {
 // May create the object even if it returns an error - if so
 // will return the object and the error, otherwise will return
 // nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	// fs.Debugf(f, "Trying to put file %s", src.Remote())
-	err := f.mkParentDir(ctx, src.Remote())
+	err := f.mkParentDir(src.Remote())
 	if err != nil {
 		return nil, errors.Wrap(err, "Put mkParentDir failed")
 	}
@@ -707,40 +399,38 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		fs:     f,
 		remote: src.Remote(),
 	}
-	err = o.Update(ctx, in, src, options...)
+	err = o.Update(in, src, options...)
 	return o, err
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(ctx, in, src, options...)
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(in, src, options...)
 }
 
 // getInfo reads the FileInfo for a path
-func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
+func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
 	dir := path.Dir(remote)
 	base := path.Base(remote)
 
-	c, err := f.getFtpConnection(ctx)
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "getInfo")
 	}
-	files, err := c.List(f.dirFromStandardPath(dir))
+	files, err := c.List(dir)
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 
 	for i := range files {
-		file := files[i]
-		f.entryToStandard(file)
-		if file.Name == base {
+		if files[i].Name == base {
 			info := &FileInfo{
 				Name:    remote,
-				Size:    file.Size,
-				ModTime: file.Time,
-				IsDir:   file.Type == ftp.EntryTypeFolder,
+				Size:    files[i].Size,
+				ModTime: files[i].Time,
+				IsDir:   files[i].Type == ftp.EntryTypeFolder,
 			}
 			return info, nil
 		}
@@ -749,12 +439,11 @@ func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err erro
 }
 
 // mkdir makes the directory and parents using unrooted paths
-func (f *Fs) mkdir(ctx context.Context, abspath string) error {
-	abspath = path.Clean(abspath)
+func (f *Fs) mkdir(abspath string) error {
 	if abspath == "." || abspath == "/" {
 		return nil
 	}
-	fi, err := f.getInfo(ctx, abspath)
+	fi, err := f.getInfo(abspath)
 	if err == nil {
 		if fi.IsDir {
 			return nil
@@ -764,15 +453,15 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 		return errors.Wrapf(err, "mkdir %q failed", abspath)
 	}
 	parent := path.Dir(abspath)
-	err = f.mkdir(ctx, parent)
+	err = f.mkdir(parent)
 	if err != nil {
 		return err
 	}
-	c, connErr := f.getFtpConnection(ctx)
+	c, connErr := f.getFtpConnection()
 	if connErr != nil {
 		return errors.Wrap(connErr, "mkdir")
 	}
-	err = c.MakeDir(f.dirFromStandardPath(abspath))
+	err = c.MakeDir(abspath)
 	f.putFtpConnection(&c, err)
 	switch errX := err.(type) {
 	case *textproto.Error:
@@ -788,55 +477,55 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 
 // mkParentDir makes the parent of remote if necessary and any
 // directories above that
-func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
+func (f *Fs) mkParentDir(remote string) error {
 	parent := path.Dir(remote)
-	return f.mkdir(ctx, path.Join(f.root, parent))
+	return f.mkdir(path.Join(f.root, parent))
 }
 
 // Mkdir creates the directory if it doesn't exist
-func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
+func (f *Fs) Mkdir(dir string) (err error) {
 	// defer fs.Trace(dir, "")("err=%v", &err)
 	root := path.Join(f.root, dir)
-	return f.mkdir(ctx, root)
+	return f.mkdir(root)
 }
 
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	c, err := f.getFtpConnection(ctx)
+func (f *Fs) Rmdir(dir string) error {
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return errors.Wrap(translateErrorFile(err), "Rmdir")
 	}
-	err = c.RemoveDir(f.dirFromStandardPath(path.Join(f.root, dir)))
+	err = c.RemoveDir(path.Join(f.root, dir))
 	f.putFtpConnection(&c, err)
 	return translateErrorDir(err)
 }
 
 // Move renames a remote file object
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't move - not same remote type")
 		return nil, fs.ErrorCantMove
 	}
-	err := f.mkParentDir(ctx, remote)
+	err := f.mkParentDir(remote)
 	if err != nil {
 		return nil, errors.Wrap(err, "Move mkParentDir failed")
 	}
-	c, err := f.getFtpConnection(ctx)
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "Move")
 	}
 	err = c.Rename(
-		f.opt.Enc.FromStandardPath(path.Join(srcObj.fs.root, srcObj.remote)),
-		f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
+		path.Join(srcObj.fs.root, srcObj.remote),
+		path.Join(f.root, remote),
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "Move Rename failed")
 	}
-	dstObj, err := f.NewObject(ctx, remote)
+	dstObj, err := f.NewObject(remote)
 	if err != nil {
 		return nil, errors.Wrap(err, "Move NewObject failed")
 	}
@@ -844,14 +533,14 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
+// using server side move operations.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
 // If it isn't possible then return fs.ErrorCantDirMove
 //
 // If destination exists then return fs.ErrorDirExists
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	srcFs, ok := src.(*Fs)
 	if !ok {
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
@@ -861,7 +550,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	dstPath := path.Join(f.root, dstRemote)
 
 	// Check if destination exists
-	fi, err := f.getInfo(ctx, dstPath)
+	fi, err := f.getInfo(dstPath)
 	if err == nil {
 		if fi.IsDir {
 			return fs.ErrorDirExists
@@ -872,19 +561,19 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	// Make sure the parent directory exists
-	err = f.mkdir(ctx, path.Dir(dstPath))
+	err = f.mkdir(path.Dir(dstPath))
 	if err != nil {
 		return errors.Wrap(err, "DirMove mkParentDir dst failed")
 	}
 
 	// Do the move
-	c, err := f.getFtpConnection(ctx)
+	c, err := f.getFtpConnection()
 	if err != nil {
 		return errors.Wrap(err, "DirMove")
 	}
 	err = c.Rename(
-		f.dirFromStandardPath(srcPath),
-		f.dirFromStandardPath(dstPath),
+		srcPath,
+		dstPath,
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -914,7 +603,7 @@ func (o *Object) Remote() string {
 }
 
 // Hash returns the hash of an object returning a lowercase hex string
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+func (o *Object) Hash(t hash.Type) (string, error) {
 	return "", hash.ErrUnsupported
 }
 
@@ -924,12 +613,12 @@ func (o *Object) Size() int64 {
 }
 
 // ModTime returns the modification time of the object
-func (o *Object) ModTime(ctx context.Context) time.Time {
+func (o *Object) ModTime() time.Time {
 	return o.info.ModTime
 }
 
 // SetModTime sets the modification time of the object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+func (o *Object) SetModTime(modTime time.Time) error {
 	return nil
 }
 
@@ -957,36 +646,18 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
-	var err error
-	errchan := make(chan error, 1)
-	go func() {
-		errchan <- f.rc.Close()
-	}()
-	// Wait for Close for up to 60 seconds by default
-	timer := time.NewTimer(time.Duration(f.f.opt.CloseTimeout))
-	select {
-	case err = <-errchan:
-		timer.Stop()
-	case <-timer.C:
-		// if timer fired assume no error but connection dead
-		fs.Errorf(f.f, "Timeout when waiting for connection Close")
-		f.f.putFtpConnection(nil, nil)
-		return nil
-	}
+	err := f.rc.Close()
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
-		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
 	}
 	// mask the error if it was caused by a premature close
-	// NB StatusAboutToSend is to work around a bug in pureftpd
-	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
 	switch errX := err.(type) {
 	case *textproto.Error:
 		switch errX.Code {
-		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
+		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable:
 			err = nil
 		}
 	}
@@ -994,7 +665,7 @@ func (f *ftpReadCloser) Close() error {
 }
 
 // Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
 	// defer fs.Trace(o, "")("rc=%v, err=%v", &rc, &err)
 	path := path.Join(o.fs.root, o.remote)
 	var offset, limit int64 = 0, -1
@@ -1010,11 +681,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+	c, err := o.fs.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
-	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+	fd, err := c.RetrFrom(path, uint64(offset))
 	if err != nil {
 		o.fs.putFtpConnection(&c, err)
 		return nil, errors.Wrap(err, "open")
@@ -1028,36 +699,30 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 // Copy the reader into the object updating modTime and size
 //
 // The new object may have been created if an error is returned
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	// defer fs.Trace(o, "src=%v", src)("err=%v", &err)
 	path := path.Join(o.fs.root, o.remote)
 	// remove the file if upload failed
 	remove := func() {
-		// Give the FTP server a chance to get its internal state in order after the error.
-		// The error may have been local in which case we closed the connection.  The server
-		// may still be dealing with it for a moment. A sleep isn't ideal but I haven't been
-		// able to think of a better method to find out if the server has finished - ncw
-		time.Sleep(1 * time.Second)
-		removeErr := o.Remove(ctx)
+		removeErr := o.Remove()
 		if removeErr != nil {
 			fs.Debugf(o, "Failed to remove: %v", removeErr)
 		} else {
 			fs.Debugf(o, "Removed after failed upload: %v", err)
 		}
 	}
-	c, err := o.fs.getFtpConnection(ctx)
+	c, err := o.fs.getFtpConnection()
 	if err != nil {
 		return errors.Wrap(err, "Update")
 	}
-	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
+	err = c.Stor(path, in)
 	if err != nil {
-		_ = c.Quit() // toss this connection to avoid sync errors
+		_ = c.Quit()
 		remove()
-		o.fs.putFtpConnection(nil, err)
 		return errors.Wrap(err, "update stor")
 	}
 	o.fs.putFtpConnection(&c, nil)
-	o.info, err = o.fs.getInfo(ctx, path)
+	o.info, err = o.fs.getInfo(path)
 	if err != nil {
 		return errors.Wrap(err, "update getinfo")
 	}
@@ -1065,22 +730,22 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove an object
-func (o *Object) Remove(ctx context.Context) (err error) {
+func (o *Object) Remove() (err error) {
 	// defer fs.Trace(o, "")("err=%v", &err)
 	path := path.Join(o.fs.root, o.remote)
 	// Check if it's a directory or a file
-	info, err := o.fs.getInfo(ctx, path)
+	info, err := o.fs.getInfo(path)
 	if err != nil {
 		return err
 	}
 	if info.IsDir {
-		err = o.fs.Rmdir(ctx, o.remote)
+		err = o.fs.Rmdir(o.remote)
 	} else {
-		c, err := o.fs.getFtpConnection(ctx)
+		c, err := o.fs.getFtpConnection()
 		if err != nil {
 			return errors.Wrap(err, "Remove")
 		}
-		err = c.Delete(o.fs.opt.Enc.FromStandardPath(path))
+		err = c.Delete(path)
 		o.fs.putFtpConnection(&c, err)
 	}
 	return err
@@ -1092,6 +757,5 @@ var (
 	_ fs.Mover       = &Fs{}
 	_ fs.DirMover    = &Fs{}
 	_ fs.PutStreamer = &Fs{}
-	_ fs.Shutdowner  = &Fs{}
 	_ fs.Object      = &Object{}
 )

backend/ftp/ftp_test.go

@@ -4,45 +4,14 @@ package ftp_test
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/ftp"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/ftp"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTPProftpd:",
+		RemoteName: "TestFTP:",
 		NilObject:  (*ftp.Object)(nil),
 	})
 }
-
-func TestIntegration2(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("skipping as -remote is set")
-	}
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTPRclone:",
-		NilObject:  (*ftp.Object)(nil),
-	})
-}
-
-func TestIntegration3(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("skipping as -remote is set")
-	}
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTPPureftpd:",
-		NilObject:  (*ftp.Object)(nil),
-	})
-}
-
-// func TestIntegration4(t *testing.T) {
-// 	if *fstest.RemoteName != "" {
-// 		t.Skip("skipping as -remote is set")
-// 	}
-// 	fstests.Run(t, &fstests.Opt{
-// 		RemoteName: "TestFTPVsftpd:",
-// 		NilObject:  (*ftp.Object)(nil),
-// 	})
-// }

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,12 +1,11 @@
 // Test GoogleCloudStorage filesystem interface
-
 package googlecloudstorage_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/googlecloudstorage"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/googlecloudstorage"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote