Version v1.48.0

Fixes problem introduced in d2be792d5e

.golangci.yml

@@ -1,9 +1,5 @@
 # golangci-lint configuration options
 
-run:
-  build-tags:
-    - cmount
-
 linters:
   enable:
     - deadcode

.travis.yml

@@ -1,54 +1,101 @@
+---
 language: go
 sudo: required
-dist: trusty
+dist: xenial
 os:
-- linux
-go:
-- 1.8.x
-- 1.9.x
-- 1.10.x
-- 1.11.x
-- 1.12rc1
-- tip
+  - linux
 go_import_path: github.com/ncw/rclone
 before_install:
-- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
-- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
+  - git fetch --unshallow --tags
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      sudo modprobe fuse
+      sudo chmod 666 /dev/fuse
+      sudo chown root:$USER /etc/fuse.conf
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+      brew update
+      brew tap caskroom/cask
+      brew cask install osxfuse
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
+      choco install -y winfsp zip make
+      cd ../..                  # fix crlf in git checkout
+      mv $TRAVIS_REPO_SLUG _old
+      git config --global core.autocrlf false
+      git clone _old $TRAVIS_REPO_SLUG
+      cd $TRAVIS_REPO_SLUG
+    fi
 install:
-- git fetch --unshallow --tags
-- make vars
-- make build_dep
-script:
-- make check
-- make quicktest
-- make compile_all
+  - make vars
 env:
   global:
     - GOTAGS=cmount
+    - GOMAXPROCS=8              # workaround for cmd/mount tests locking up - see #3154
     - GO111MODULE=off
+    - GITHUB_USER=ncw
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
-    - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-    - fuse
-    - libfuse-dev
-    - rpm
-    - pkg-config
+      - fuse
+      - libfuse-dev
+      - rpm
+      - pkg-config
 cache:
   directories:
     - $HOME/.cache/go-build
 matrix:
   allow_failures:
-  - go: tip
+    - go: tip
   include:
-  - os: osx
-    go: 1.12rc1
-    env: GOTAGS=""
-    cache:
-      directories:
-        - $HOME/Library/Caches/go-build
+    - go: 1.9.x
+      script:
+        - make quicktest
+    - go: 1.10.x
+      script:
+        - make quicktest
+    - go: 1.11.x
+      script:
+        - make quicktest
+    - go: 1.12.x
+      env:
+        - GOTAGS=cmount
+      script:
+        - make build_dep
+        - make check
+        - make quicktest
+        - make racequicktest
+        - make compile_all
+    - os: osx
+      go: 1.12.x
+      env:
+        - GOTAGS=  # cmount doesn't work on osx travis for some reason
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+      script:
+        - make
+        - make quicktest
+        - make racequicktest
+    # - os: windows
+    #   go: 1.12.x
+    #   env:
+    #     - GOTAGS=cmount
+    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
+    #   #filter_secrets: false     # works around a problem with secrets under windows
+    #   cache:
+    #     directories:
+    #       - ${LocalAppData}/go-build
+    #   script:
+    #     - make
+    #     - make quicktest
+    #     - make racequicktest
+    - go: tip
+      script:
+        - make quicktest
+
 deploy:
   provider: script
   script: make travis_beta
@@ -56,5 +103,5 @@ deploy:
   on:
     repo: ncw/rclone
     all_branches: true
-    go: 1.12rc1
-    condition: $TRAVIS_PULL_REQUEST == false
+    go: 1.12.x
+    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"

CONTRIBUTING.md

@@ -135,7 +135,7 @@ then change into the project root and run
 
     make test
 
-This command is run daily on the the integration test server. You can
+This command is run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##

Makefile

@@ -17,8 +17,6 @@ ifneq ($(TAG),$(LAST_TAG))
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
-# Run full tests if go >= go1.11
-FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 11)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -26,6 +24,7 @@ BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
+LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
 .PHONY: rclone vars version
@@ -42,7 +41,6 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
-	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -57,28 +55,19 @@ test:	rclone
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-ifdef FULL_TESTS
+
+racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
-endif
 
 # Do source code quality checks
 check:	rclone
-ifdef FULL_TESTS
-	@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
-	@# see: https://github.com/golangci/golangci-lint/issues/204
 	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
-	@golangci-lint run ./...
+	@golangci-lint run $(LINTTAGS) ./...
 	@echo "-- END CODE QUALITY REPORT ---------------------------------"
-else
-	@echo Skipping source quality tests as version of go too old
-endif
 
 # Get the build dependencies
 build_dep:
-ifdef FULL_TESTS
 	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
-endif
 
 # Get the release dependencies
 release_dep:
@@ -109,7 +98,7 @@ commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
 
 backenddocs: rclone bin/make_backend_docs.py
-	./bin/make_backend_docs.py
+	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -162,11 +151,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
-else
-	@echo Skipping compile all as version of go too old
-endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -186,6 +171,11 @@ BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
+ifeq ($(TRAVIS_OS_NAME),windows)
+# BUILD_FLAGS := -include "^windows/" -cgo
+# 386 doesn't build yet
+	BUILD_FLAGS := -include "^windows/amd64" -cgo
+endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
@@ -201,7 +191,7 @@ endif
 
 # Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync $(BETA_UPLOAD) build/
+	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server -v -w

README.md

@@ -7,11 +7,11 @@
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/) | 
-[G+](https://google.com/+RcloneOrg)
 
 [![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
 [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
 [![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/ncw/rclone)](https://goreportcard.com/report/github.com/ncw/rclone)
 [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
 # Rclone
@@ -36,6 +36,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
@@ -72,6 +73,8 @@ Please see [the full list of all storage providers and their features](https://r
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
+  * Multi-threaded downloads to local disk
+  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
 
 ## Installation & documentation
 

RELEASE.md

@@ -11,7 +11,7 @@ Making a release
   * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX"
+  * git commit -a -v -m "Version v1.XX.0"
   * make retag
   * git push --tags origin master
   * # Wait for the appveyor and travis builds to complete then...
@@ -27,6 +27,7 @@ Making a release
 
 Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
+      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files

backend/adb/adb.go

@@ -1,821 +0,0 @@
-package adb
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config/configmap"
-	"github.com/ncw/rclone/fs/config/configstruct"
-	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/readers"
-	"github.com/pkg/errors"
-	adb "github.com/thinkhy/go-adb"
-	"github.com/thinkhy/go-adb/wire"
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "adb",
-		Description: "Android Debug Bridge",
-		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Name:     "serial",
-			Help:     "The device serial to use. Leave empty for auto selection.",
-			Advanced: true,
-		}, {
-			Name:     "host",
-			Default:  "localhost",
-			Help:     "The ADB server host.",
-			Advanced: true,
-		}, {
-			Name:     "port",
-			Default:  5037,
-			Help:     "The ADB server port.",
-			Advanced: true,
-		}, {
-			Name:     "executable",
-			Help:     "The ADB executable path.",
-			Advanced: true,
-		}, {
-			Name:     "copy_links",
-			Help:     "Follow symlinks and copy the pointed to item.",
-			Default:  false,
-			Advanced: true,
-		}},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	Serial         string
-	Host           string
-	Port           uint16
-	Executable     string
-	FollowSymlinks bool `config:"copy_links"`
-}
-
-// Fs represents a adb device
-type Fs struct {
-	name        string       // name of this remote
-	root        string       // the path we are working on
-	opt         Options      // parsed options
-	features    *fs.Features // optional features
-	client      *adb.Adb
-	device      *execDevice
-	statFunc    statFunc
-	statFuncMu  sync.Mutex
-	touchFunc   touchFunc
-	touchFuncMu sync.Mutex
-}
-
-// Object describes a adb file
-type Object struct {
-	fs      *Fs    // what this object is part of
-	remote  string // The remote path
-	size    int64
-	mode    os.FileMode
-	modTime time.Time
-}
-
-// ------------------------------------------------------------
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("ADB root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	// Parse config into Options struct
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	if root == "" {
-		root = "/"
-	}
-
-	f := &Fs{
-		name:      name,
-		root:      root,
-		opt:       *opt,
-		statFunc:  (*Object).statTry,
-		touchFunc: (*Object).touchTry,
-	}
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-	}).Fill(f)
-
-	f.client, err = adb.NewWithConfig(adb.ServerConfig{
-		Host:      opt.Host,
-		Port:      int(opt.Port),
-		PathToAdb: opt.Executable,
-	})
-	if err != nil {
-		return nil, errors.Wrapf(err, "Could not configure ADB server")
-	}
-	err = f.client.StartServer()
-	if err != nil {
-		return nil, errors.Wrapf(err, "Could not start ADB server")
-	}
-
-	serverVersion, err := f.client.ServerVersion()
-	if err != nil {
-		return nil, errors.Wrapf(err, "Could not get ADB server version")
-	}
-	fs.Debugf(f, "ADB server version: 0x%X", serverVersion)
-
-	serials, err := f.client.ListDeviceSerials()
-	if err != nil {
-		return nil, errors.Wrapf(err, "Could not get ADB devices")
-	}
-	descriptor := adb.AnyDevice()
-	if opt.Serial != "" {
-		descriptor = adb.DeviceWithSerial(opt.Serial)
-	}
-	if len(serials) > 1 && opt.Serial == "" {
-		return nil, errors.New("Multiple ADB devices found. Use the serial config to select a specific device")
-	}
-	f.device = &execDevice{f.client.Device(descriptor)}
-
-	// follow symlinks for root pathes
-	entry, err := f.newEntryFollowSymlinks("")
-	switch err {
-	case nil:
-	case fs.ErrorObjectNotFound:
-	default:
-		return nil, err
-	}
-	switch entry.(type) {
-	case fs.Object:
-		f.root = path.Dir(f.root)
-		return f, fs.ErrorIsFile
-	case nil:
-		return f, nil
-	case fs.Directory:
-		return f, nil
-	default:
-		return nil, errors.Errorf("Invalid root entry type %t", entry)
-	}
-}
-
-// Precision of the object storage system
-func (f *Fs) Precision() time.Duration {
-	return 1 * time.Second
-}
-
-// Hashes returns the supported hash sets.
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.None)
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	p := path.Join(f.root, dir)
-	dirEntries, err := f.device.ListDirEntries(p)
-	if err != nil {
-		return nil, errors.Wrap(err, "ListDirEntries")
-	}
-
-	defer fs.CheckClose(dirEntries, &err)
-
-	found := false
-	for dirEntries.Next() {
-		found = true
-		dirEntry := dirEntries.Entry()
-		switch dirEntry.Name {
-		case ".", "..":
-			continue
-		}
-		fsEntry, err := f.entryForDirEntry(path.Join(dir, dirEntry.Name), dirEntry, f.opt.FollowSymlinks)
-		if err != nil {
-			fs.Errorf(p, "Listing error: %q: %v", dirEntry.Name, err)
-			return nil, err
-		} else if fsEntry != nil {
-			entries = append(entries, fsEntry)
-		} else {
-			fs.Debugf(f, "Skipping DirEntry %#v", dirEntry)
-		}
-	}
-	err = dirEntries.Err()
-	if err != nil {
-		return nil, errors.Wrap(err, "ListDirEntries")
-	}
-	if !found {
-		return nil, fs.ErrorDirNotFound
-	}
-	return
-}
-
-func (f *Fs) entryForDirEntry(remote string, e *adb.DirEntry, followSymlinks bool) (fs.DirEntry, error) {
-	o := f.newObjectWithInfo(remote, e)
-	// Follow symlinks if required
-	if followSymlinks && (e.Mode&os.ModeSymlink) != 0 {
-		err := f.statFunc(&o)
-		if err != nil {
-			return nil, err
-		}
-	}
-	if o.mode.IsDir() {
-		return fs.NewDir(remote, o.modTime), nil
-	}
-	return &o, nil
-}
-
-func (f *Fs) newEntry(remote string) (fs.DirEntry, error) {
-	return f.newEntryWithFollow(remote, f.opt.FollowSymlinks)
-}
-func (f *Fs) newEntryFollowSymlinks(remote string) (fs.DirEntry, error) {
-	return f.newEntryWithFollow(remote, true)
-}
-func (f *Fs) newEntryWithFollow(remote string, followSymlinks bool) (fs.DirEntry, error) {
-	entry, err := f.device.Stat(path.Join(f.root, remote))
-	if err != nil {
-		if adb.HasErrCode(err, adb.FileNoExistError) {
-			return nil, fs.ErrorObjectNotFound
-		}
-		return nil, errors.Wrapf(err, "Stat failed")
-	}
-	return f.entryForDirEntry(remote, entry, followSymlinks)
-}
-
-func (f *Fs) newObjectWithInfo(remote string, e *adb.DirEntry) Object {
-	return Object{
-		fs:      f,
-		remote:  remote,
-		size:    int64(e.Size),
-		mode:    e.Mode,
-		modTime: e.ModifiedAt,
-	}
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error ErrorObjectNotFound.
-func (f *Fs) NewObject(remote string) (fs.Object, error) {
-	entry, err := f.newEntry(remote)
-	if err != nil {
-		return nil, err
-	}
-	obj, ok := entry.(fs.Object)
-	if !ok {
-		return nil, fs.ErrorObjectNotFound
-	}
-	return obj, nil
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
-// return an error or upload it properly (rather than e.g. calling panic).
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	remote := src.Remote()
-	// Temporary Object under construction - info filled in by Update()
-	o := f.newObject(remote)
-	err := o.Update(in, src, options...)
-	if err != nil {
-		return nil, err
-	}
-	return o, nil
-}
-
-// newObject makes a half completed Object
-func (f *Fs) newObject(remote string) *Object {
-	return &Object{
-		fs:     f,
-		remote: remote,
-	}
-}
-
-// Mkdir makes the directory (container, bucket)
-//
-// Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(dir string) error {
-	p := path.Join(f.root, dir)
-	output, code, err := f.device.execCommandWithExitCode("mkdir -p", p)
-	switch err := err.(type) {
-	case nil:
-		return nil
-	case adb.ShellExitError:
-		entry, _ := f.newEntry(p)
-		if _, ok := entry.(fs.Directory); ok {
-			return nil
-		}
-		return errors.Errorf("mkdir %q failed with %d: %q", dir, code, output)
-	default:
-		return errors.Wrap(err, "mkdir")
-	}
-}
-
-// Rmdir removes the directory (container, bucket) if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(dir string) error {
-	p := path.Join(f.root, dir)
-	output, code, err := f.device.execCommandWithExitCode("rmdir", p)
-	switch err := err.(type) {
-	case nil:
-		return nil
-	case adb.ShellExitError:
-		return errors.Errorf("rmdir %q failed with %d: %q", dir, code, output)
-	default:
-		return errors.Wrap(err, "rmdir")
-	}
-}
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification date of the file
-// It should return a best guess if one isn't available
-func (o *Object) ModTime() time.Time {
-	return o.modTime
-}
-
-// Size returns the size of the file
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// Hash returns the selected checksum of the file
-// If no checksum is available it returns ""
-func (o *Object) Hash(hash.Type) (string, error) {
-	return "", hash.ErrUnsupported
-}
-
-// Storable says whether this object can be stored
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the metadata on the object to set the modification date
-func (o *Object) SetModTime(t time.Time) error {
-	return o.fs.touchFunc(o, t)
-}
-
-func (o *Object) stat() error {
-	return o.statStatArg(statArgC, path.Join(o.fs.root, o.remote))
-}
-
-func (o *Object) setMetadata(entry *adb.DirEntry) {
-	// Don't overwrite the values if we don't need to
-	// this avoids upsetting the race detector
-	if o.size != int64(entry.Size) {
-		o.size = int64(entry.Size)
-	}
-	if !o.modTime.Equal(entry.ModifiedAt) {
-		o.modTime = entry.ModifiedAt
-	}
-	if o.mode != entry.Mode {
-		o.mode = decodeEntryMode(uint32(entry.Mode))
-	}
-}
-
-// Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
-	const blockSize = 1 << 12
-
-	var offset, count int64 = 0, -1
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-		case *fs.SeekOption:
-			offset = x.Offset
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-	if offset > o.size {
-		offset = o.size
-	}
-	if count < 0 {
-		count = o.size - offset
-	} else if count+offset > o.size {
-		count = o.size - offset
-	}
-	fs.Debugf(o, "Open: remote: %q offset: %d count: %d", o.remote, offset, count)
-
-	if count == 0 {
-		return ioutil.NopCloser(bytes.NewReader(nil)), nil
-	}
-	offsetBlocks, offsetRest := offset/blockSize, offset%blockSize
-	countBlocks := (count-1)/blockSize + 1
-
-	conn, err := o.fs.device.execCommand(fmt.Sprintf("sh -c 'dd \"if=$0\" bs=%d skip=%d count=%d 2>/dev/null'", blockSize, offsetBlocks, countBlocks), path.Join(o.fs.root, o.remote))
-	if err != nil {
-		return nil, err
-	}
-
-	return &adbReader{
-		ReadCloser: readers.NewLimitedReadCloser(conn, count+offsetRest),
-		skip:       offsetRest,
-		expected:   count,
-	}, nil
-}
-
-// Update in to the object with the modTime given of the given size
-//
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
-// return an error or update the object properly (rather than e.g. calling panic).
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	for _, option := range options {
-		if option.Mandatory() {
-			fs.Logf(option, "Unsupported mandatory option: %v", option)
-		}
-	}
-	written, err := o.writeToFile(path.Join(o.fs.root, o.remote), in, 0666, src.ModTime())
-	if err != nil {
-		if removeErr := o.Remove(); removeErr != nil {
-			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
-		}
-		return err
-	}
-	expected := src.Size()
-	if expected == -1 {
-		expected = written
-	}
-	for _, t := range []int64{100, 250, 500, 1000, 2500, 5000, 10000} {
-		err = o.stat()
-		if err != nil {
-			return err
-		}
-		if o.size == expected {
-			return nil
-		}
-		fs.Debugf(o, "Invalid size after update, expected: %d got: %d", expected, o.size)
-		time.Sleep(time.Duration(t) * time.Millisecond)
-	}
-	return o.stat()
-}
-
-// Remove this object
-func (o *Object) Remove() error {
-	p := path.Join(o.fs.root, o.remote)
-	output, code, err := o.fs.device.execCommandWithExitCode("rm", p)
-	switch err := err.(type) {
-	case nil:
-		return nil
-	case adb.ShellExitError:
-		return errors.Errorf("rm %q failed with %d: %q", o.remote, code, output)
-	default:
-		return errors.Wrap(err, "rm")
-	}
-}
-
-func (o *Object) writeToFile(path string, rd io.Reader, perms os.FileMode, modeTime time.Time) (written int64, err error) {
-	dst, err := o.fs.device.OpenWrite(path, perms, modeTime)
-	if err != nil {
-		return
-	}
-	defer fs.CheckClose(dst, &err)
-	return io.Copy(dst, rd)
-}
-
-type statFunc func(*Object) error
-
-func (o *Object) statTry() error {
-	o.fs.statFuncMu.Lock()
-	defer o.fs.statFuncMu.Unlock()
-
-	for _, f := range []statFunc{
-		(*Object).statStatL, (*Object).statRealPath, (*Object).statReadLink,
-	} {
-		err := f(o)
-		if err != nil {
-			fs.Debugf(o, "%s", err)
-		} else {
-			o.fs.statFunc = f
-			return nil
-		}
-	}
-
-	return errors.Errorf("unable to resolve link target")
-}
-
-const (
-	statArgLc = "-Lc"
-	statArgC  = "-c"
-)
-
-func (o *Object) statStatL() error {
-	return o.statStatArg(statArgLc, path.Join(o.fs.root, o.remote))
-}
-
-func (o *Object) statStatArg(arg, path string) error {
-	output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("stat %s %s", arg, "%f,%s,%Y"), path)
-	output = strings.TrimSpace(output)
-	switch err := err.(type) {
-	case nil:
-	case adb.ShellExitError:
-		return errors.Errorf("stat %q failed with %d: %q", o.remote, code, output)
-	default:
-		return errors.Wrap(err, "stat")
-	}
-
-	parts := strings.Split(output, ",")
-	if len(parts) != 3 {
-		return errors.Errorf("stat %q invalid output %q", o.remote, output)
-	}
-
-	mode, err := strconv.ParseUint(parts[0], 16, 32)
-	if err != nil {
-		return errors.Errorf("stat %q invalid output %q", o.remote, output)
-	}
-	size, err := strconv.ParseUint(parts[1], 10, 64)
-	if err != nil {
-		return errors.Errorf("stat %q invalid output %q", o.remote, output)
-	}
-	modTime, err := strconv.ParseInt(parts[2], 10, 64)
-	if err != nil {
-		return errors.Errorf("stat %q invalid output %q", o.remote, output)
-	}
-
-	o.size = int64(size)
-	o.modTime = time.Unix(modTime, 0)
-	o.mode = decodeEntryMode(uint32(mode))
-	return nil
-}
-
-func (o *Object) statReadLink() error {
-	p := path.Join(o.fs.root, o.remote)
-	output, code, err := o.fs.device.execCommandWithExitCode("readlink -f", p)
-	output = strings.TrimSuffix(output, "\n")
-	switch err := err.(type) {
-	case nil:
-	case adb.ShellExitError:
-		return errors.Errorf("readlink %q failed with %d: %q", o.remote, code, output)
-	default:
-		return errors.Wrap(err, "readlink")
-	}
-	return o.statStatArg(statArgC, output)
-}
-func (o *Object) statRealPath() error {
-	p := path.Join(o.fs.root, o.remote)
-	output, code, err := o.fs.device.execCommandWithExitCode("realpath", p)
-	output = strings.TrimSuffix(output, "\n")
-	switch err := err.(type) {
-	case nil:
-	case adb.ShellExitError:
-		return errors.Errorf("realpath %q failed with %d: %q", o.remote, code, output)
-	default:
-		return errors.Wrap(err, "realpath")
-	}
-	return o.statStatArg(statArgC, output)
-}
-
-type touchFunc func(*Object, time.Time) error
-
-func (o *Object) touchTry(t time.Time) error {
-	o.fs.touchFuncMu.Lock()
-	defer o.fs.touchFuncMu.Unlock()
-
-	for _, f := range []touchFunc{
-		(*Object).touchCmd, (*Object).touchCd,
-	} {
-		err := f(o, t)
-		if err != nil {
-			fs.Debugf(o, "%s", err)
-		} else {
-			o.fs.touchFunc = f
-			return nil
-		}
-	}
-
-	return errors.Errorf("unable to resolve link target")
-}
-
-const (
-	touchArgCmd = "-cmd"
-	touchArgCd  = "-cd"
-)
-
-func (o *Object) touchCmd(t time.Time) error {
-	return o.touchStatArg(touchArgCmd, path.Join(o.fs.root, o.remote), t)
-}
-func (o *Object) touchCd(t time.Time) error {
-	return o.touchStatArg(touchArgCd, path.Join(o.fs.root, o.remote), t)
-}
-
-func (o *Object) touchStatArg(arg, path string, t time.Time) error {
-	output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("touch %s %s", arg, t.Format(time.RFC3339Nano)), path)
-	output = strings.TrimSpace(output)
-	switch err := err.(type) {
-	case nil:
-	case adb.ShellExitError:
-		return errors.Errorf("touch %q failed with %d: %q", o.remote, code, output)
-	default:
-		return errors.Wrap(err, "touch")
-	}
-
-	err = o.stat()
-	if err != nil {
-		return err
-	}
-	if diff, ok := checkTimeEqualWithPrecision(t, o.modTime, o.fs.Precision()); !ok {
-		return errors.Errorf("touch %q to %s was ineffective: %d", o.remote, t, diff)
-	}
-
-	return nil
-}
-
-func checkTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
-	dt := t0.Sub(t1)
-	if dt >= precision || dt <= -precision {
-		return dt, false
-	}
-	return dt, true
-}
-
-func decodeEntryMode(entryMode uint32) os.FileMode {
-	const (
-		unixIFBLK  = 0x6000
-		unixIFMT   = 0xf000
-		unixIFCHR  = 0x2000
-		unixIFDIR  = 0x4000
-		unixIFIFO  = 0x1000
-		unixIFLNK  = 0xa000
-		unixIFREG  = 0x8000
-		unixIFSOCK = 0xc000
-		unixISGID  = 0x400
-		unixISUID  = 0x800
-		unixISVTX  = 0x200
-	)
-
-	mode := os.FileMode(entryMode & 0777)
-	switch entryMode & unixIFMT {
-	case unixIFBLK:
-		mode |= os.ModeDevice
-	case unixIFCHR:
-		mode |= os.ModeDevice | os.ModeCharDevice
-	case unixIFDIR:
-		mode |= os.ModeDir
-	case unixIFIFO:
-		mode |= os.ModeNamedPipe
-	case unixIFLNK:
-		mode |= os.ModeSymlink
-	case unixIFREG:
-		// nothing to do
-	case unixIFSOCK:
-		mode |= os.ModeSocket
-	}
-	if entryMode&unixISGID != 0 {
-		mode |= os.ModeSetgid
-	}
-	if entryMode&unixISUID != 0 {
-		mode |= os.ModeSetuid
-	}
-	if entryMode&unixISVTX != 0 {
-		mode |= os.ModeSticky
-	}
-	return mode
-}
-
-type execDevice struct {
-	*adb.Device
-}
-
-func (d *execDevice) execCommandWithExitCode(cmd string, arg string) (string, int, error) {
-	cmdLine := fmt.Sprintf("sh -c '%s \"$0\"; echo :$?' '%s'", cmd, strings.Replace(arg, "'", "'\\''", -1))
-	fs.Debugf("adb", "exec: %s", cmdLine)
-	conn, err := d.execCommand(cmdLine)
-	if err != nil {
-		return "", -1, err
-	}
-
-	resp, err := conn.ReadUntilEof()
-	if err != nil {
-		return "", -1, errors.Wrap(err, "ExecCommand")
-	}
-
-	outStr := string(resp)
-	idx := strings.LastIndexByte(outStr, ':')
-	if idx == -1 {
-		return outStr, -1, fmt.Errorf("adb shell aborted, can not parse exit code")
-	}
-	exitCode, _ := strconv.Atoi(strings.TrimSpace(outStr[idx+1:]))
-	if exitCode != 0 {
-		err = adb.ShellExitError{Command: cmdLine, ExitCode: exitCode}
-	}
-	return outStr[:idx], exitCode, err
-}
-
-func (d *execDevice) execCommand(cmd string, args ...string) (*wire.Conn, error) {
-	cmd = prepareCommandLineEscaped(cmd, args...)
-	conn, err := d.Dial()
-	if err != nil {
-		return nil, errors.Wrap(err, "ExecCommand")
-	}
-	defer func() {
-		if err != nil && conn != nil {
-			_ = conn.Close()
-		}
-	}()
-
-	req := fmt.Sprintf("exec:%s", cmd)
-
-	if err = conn.SendMessage([]byte(req)); err != nil {
-		return nil, errors.Wrap(err, "ExecCommand")
-	}
-	if _, err = conn.ReadStatus(req); err != nil {
-		return nil, errors.Wrap(err, "ExecCommand")
-	}
-	return conn, nil
-}
-
-func prepareCommandLineEscaped(cmd string, args ...string) string {
-	for i, arg := range args {
-		args[i] = fmt.Sprintf("'%s'", strings.Replace(arg, "'", "'\\''", -1))
-	}
-
-	// Prepend the command to the args array.
-	if len(args) > 0 {
-		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
-	}
-
-	return cmd
-}
-
-type adbReader struct {
-	io.ReadCloser
-	skip     int64
-	read     int64
-	expected int64
-}
-
-func (r *adbReader) Read(b []byte) (n int, err error) {
-	n, err = r.ReadCloser.Read(b)
-	if s := r.skip; n > 0 && s > 0 {
-		_n := int64(n)
-		if _n <= s {
-			r.skip -= _n
-			return r.Read(b)
-		}
-		r.skip = 0
-		copy(b, b[s:n])
-		n -= int(s)
-	}
-	r.read += int64(n)
-	if err == io.EOF && r.read < r.expected {
-		fs.Debugf("adb", "Read: read: %d expected: %d n: %d", r.read, r.expected, n)
-		return n, io.ErrUnexpectedEOF
-	}
-	return n, err
-}

backend/adb/adb_test.go

@@ -1,20 +0,0 @@
-// Test ADB filesystem interface
-package adb_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/backend/adb"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestAdb:/data/local/tmp",
-		NilObject:  (*adb.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: "TestAdb", Key: "copy_links", Value: "true"},
-		},
-	})
-}

backend/alias/alias.go

@@ -14,7 +14,7 @@ import (
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "alias",
-		Description: "Alias for a existing remote",
+		Description: "Alias for an existing remote",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "remote",

backend/all/all.go

@@ -2,7 +2,6 @@ package all
 
 import (
 	// Active file systems
-	_ "github.com/ncw/rclone/backend/adb"
 	_ "github.com/ncw/rclone/backend/alias"
 	_ "github.com/ncw/rclone/backend/amazonclouddrive"
 	_ "github.com/ncw/rclone/backend/azureblob"
@@ -17,6 +16,7 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/jottacloud"
+	_ "github.com/ncw/rclone/backend/koofr"
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -155,7 +154,7 @@ type Fs struct {
 	noAuthClient *http.Client       // unauthenticated http client
 	root         string             // the path we are working on
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	trueRootID   string             // ID of true root directory
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
@@ -273,7 +272,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:         root,
 		opt:          *opt,
 		c:            c,
-		pacer:        pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
+		pacer:        fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
 		noAuthClient: fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1093,7 +1092,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		if !bigObject {
 			in, resp, err = file.OpenHeaders(headers)
 		} else {
-			in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers)
+			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
 		return o.fs.shouldRetry(resp, err)
 	})

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 
@@ -144,7 +144,7 @@ type Fs struct {
 	containerOKMu    sync.Mutex            // mutex to protect container OK
 	containerOK      bool                  // true if we have created the container
 	containerDeleted bool                  // true if we have deleted the container
-	pacer            *pacer.Pacer          // To pace and retry the API calls
+	pacer            *fs.Pacer             // To pace and retry the API calls
 	uploadToken      *pacer.TokenDispenser // control concurrency
 }
 
@@ -347,7 +347,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:         *opt,
 		container:   container,
 		root:        directory,
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetPacer(pacer.S3Pacer),
+		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 		client:      fshttp.NewClient(fs.Config),
 	}
@@ -1386,16 +1386,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	blob := o.getBlobReference()
 	httpHeaders := azblob.BlobHTTPHeaders{}
 	httpHeaders.ContentType = fs.MimeType(o)
-	// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
-	// MD5 only for PutBlob requests
-	if size < int64(o.fs.opt.UploadCutoff) {
-		if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
-			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-			if err == nil {
-				httpHeaders.ContentMD5 = sourceMD5bytes
-			} else {
-				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
-			}
+	// Compute the Content-MD5 of the file, for multiparts uploads it
+	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
+	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
+	// 		 in order to validate its integrity during transport
+	if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
+		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+		if err == nil {
+			httpHeaders.ContentMD5 = sourceMD5bytes
+		} else {
+			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
 		}
 	}
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris !go1.8
+// +build plan9 solaris
 
 package azureblob

backend/b2/api/types.go

@@ -311,3 +311,13 @@ type CancelLargeFileResponse struct {
 	AccountID string `json:"accountId"` // The identifier for the account.
 	BucketID  string `json:"bucketId"`  // The unique ID of the bucket.
 }
+
+// CopyFileRequest is as passed to b2_copy_file
+type CopyFileRequest struct {
+	SourceID          string            `json:"sourceFileId"`                // The ID of the source file being copied.
+	Name              string            `json:"fileName"`                    // The name of the new file being created.
+	Range             string            `json:"range,omitempty"`             // The range of bytes to copy. If not provided, the whole source file will be copied.
+	MetadataDirective string            `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
+	ContentType       string            `json:"contentType,omitempty"`       // The MIME type of the content of the file (REPLACE only)
+	Info              map[string]string `json:"fileInfo,omitempty"`          // This field stores the metadata that will be stored with the file. (REPLACE only)
+}

backend/b2/b2.go

@@ -167,7 +167,7 @@ type Fs struct {
 	uploadMu      sync.Mutex                   // lock for upload variable
 	uploads       []*api.GetUploadURLResponse  // result of get upload URL calls
 	authMu        sync.Mutex                   // lock for authorizing the account
-	pacer         *pacer.Pacer                 // To pace and retry the API calls
+	pacer         *fs.Pacer                    // To pace and retry the API calls
 	bufferTokens  chan []byte                  // control concurrency of multipart uploads
 }
 
@@ -251,13 +251,7 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 				fs.Errorf(f, "Malformed %s header %q: %v", retryAfterHeader, retryAfterString, err)
 			}
 		}
-		retryAfterDuration := time.Duration(retryAfter) * time.Second
-		if f.pacer.GetSleep() < retryAfterDuration {
-			fs.Debugf(f, "Setting sleep to %v after error: %v", retryAfterDuration, err)
-			// We set 1/2 the value here because the pacer will double it immediately
-			f.pacer.SetSleep(retryAfterDuration / 2)
-		}
-		return true, err
+		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Second)
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
@@ -363,7 +357,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		srv:    rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -958,6 +952,13 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
+		if apiErr, ok := err.(*api.Error); ok {
+			if apiErr.Code == "already_hidden" {
+				// sometimes eventual consistency causes this, so
+				// ignore this error since it is harmless
+				return nil
+			}
+		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1065,6 +1066,58 @@ func (f *Fs) CleanUp() error {
 	return f.purge(true)
 }
 
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+	err := f.Mkdir("")
+	if err != nil {
+		return nil, err
+	}
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	srcFs := srcObj.fs
+	if srcFs.bucket != f.bucket {
+		fs.Debugf(src, "Can't copy - not same bucket")
+		return nil, fs.ErrorCantCopy
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_copy_file",
+	}
+	var request = api.CopyFileRequest{
+		SourceID:          srcObj.id,
+		Name:              f.root + remote,
+		MetadataDirective: "COPY",
+	}
+	var response api.FileInfo
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(&opts, &request, &response)
+		return f.shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	err = o.decodeMetaDataFileInfo(&response)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.SHA1)
@@ -1153,17 +1206,8 @@ func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
 
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
-func (o *Object) readMetaData() (err error) {
-	if o.id != "" {
-		return nil
-	}
+// getMetaData gets the metadata from the object unconditionally
+func (o *Object) getMetaData() (info *api.File, err error) {
 	maxSearched := 1
 	var timestamp api.Timestamp
 	baseRemote := o.remote
@@ -1171,7 +1215,6 @@ func (o *Object) readMetaData() (err error) {
 		timestamp, baseRemote = api.RemoveVersion(baseRemote)
 		maxSearched = maxVersions
 	}
-	var info *api.File
 	err = o.fs.list("", true, baseRemote, maxSearched, o.fs.opt.Versions, func(remote string, object *api.File, isDirectory bool) error {
 		if isDirectory {
 			return nil
@@ -1186,12 +1229,30 @@ func (o *Object) readMetaData() (err error) {
 	})
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
-			return fs.ErrorObjectNotFound
+			return nil, fs.ErrorObjectNotFound
 		}
-		return err
+		return nil, err
 	}
 	if info == nil {
-		return fs.ErrorObjectNotFound
+		return nil, fs.ErrorObjectNotFound
+	}
+	return info, nil
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// Sets
+//  o.id
+//  o.modTime
+//  o.size
+//  o.sha1
+func (o *Object) readMetaData() (err error) {
+	if o.id != "" {
+		return nil
+	}
+	info, err := o.getMetaData()
+	if err != nil {
+		return err
 	}
 	return o.decodeMetaData(info)
 }
@@ -1212,7 +1273,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return err
+		return nil
 	}
 	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
 	return nil
@@ -1230,10 +1291,33 @@ func (o *Object) ModTime() (result time.Time) {
 	return o.modTime
 }
 
-// SetModTime sets the modification time of the local fs object
+// SetModTime sets the modification time of the Object
 func (o *Object) SetModTime(modTime time.Time) error {
-	// Not possible with B2
-	return fs.ErrorCantSetModTime
+	info, err := o.getMetaData()
+	if err != nil {
+		return err
+	}
+	info.Info[timeKey] = timeString(modTime)
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_copy_file",
+	}
+	var request = api.CopyFileRequest{
+		SourceID:          o.id,
+		Name:              o.fs.root + o.remote, // copy to same name
+		MetadataDirective: "REPLACE",
+		ContentType:       info.ContentType,
+		Info:              info.Info,
+	}
+	var response api.FileInfo
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err := o.fs.srv.CallJSON(&opts, &request, &response)
+		return o.fs.shouldRetry(resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	return o.decodeMetaDataFileInfo(&response)
 }
 
 // Storable returns if this object is storable
@@ -1566,6 +1650,7 @@ func (o *Object) ID() string {
 var (
 	_ fs.Fs          = &Fs{}
 	_ fs.Purger      = &Fs{}
+	_ fs.Copier      = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.CleanUpper  = &Fs{}
 	_ fs.ListRer     = &Fs{}

backend/box/box.go

@@ -111,7 +111,7 @@ type Fs struct {
 	features     *fs.Features          // optional features
 	srv          *rest.Client          // the connection to the one drive server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *pacer.Pacer          // pacer for API calls
+	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
 	uploadToken  *pacer.TokenDispenser // control concurrency
 }
@@ -260,7 +260,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:        root,
 		opt:         *opt,
 		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
 	f.features = (&fs.Features{

backend/box/upload.go

@@ -112,7 +112,7 @@ outer:
 			return shouldRetry(resp, err)
 		})
 		delay := defaultDelay
-		why := "unknown"
+		var why string
 		if err != nil {
 			// Sometimes we get 400 Error with
 			// parts_mismatch immediately after uploading

backend/cache/cache.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ncw/rclone/backend/crypt"
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
@@ -481,7 +482,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath)
 		}
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
-		f.tempFs, err = fs.NewFs(f.opt.TempWritePath)
+		f.tempFs, err = cache.Get(f.opt.TempWritePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to create temp fs: %v", err)
 		}
@@ -1191,7 +1192,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1287,7 +1288,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)

backend/cache/cache_test.go

@@ -15,7 +15,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestCache:",
-		NilObject:  (*cache.Object)(nil),
+		RemoteName:                   "TestCache:",
+		NilObject:                    (*cache.Object)(nil),
+		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 	})
 }

backend/cache/storage_persistent.go

@@ -1023,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)

backend/crypt/crypt.go

@@ -169,23 +169,10 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
+		SetTier:                 true,
+		GetTier:                 true,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
-	doChangeNotify := wrappedFs.Features().ChangeNotify
-	if doChangeNotify != nil {
-		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
-			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-				decrypted, err := f.DecryptFileName(path)
-				if err != nil {
-					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-					return
-				}
-				notifyFunc(decrypted, entryType)
-			}
-			doChangeNotify(wrappedNotifyFunc, pollInterval)
-		}
-	}
-
 	return f, err
 }
 
@@ -202,6 +189,7 @@ type Options struct {
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
+	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
@@ -544,6 +532,16 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -616,6 +614,75 @@ func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr
 	return m.Sums()[hashType], nil
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	out := make([]fs.Directory, len(dirs))
+	for i, dir := range dirs {
+		out[i] = fs.NewDirCopy(dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
+	}
+	return do(out)
+}
+
+// DirCacheFlush resets the directory cache - used in testing
+// as an optional interface
+func (f *Fs) DirCacheFlush() {
+	do := f.Fs.Features().DirCacheFlush
+	if do != nil {
+		do()
+	}
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(remote string) (string, error) {
+	do := f.Fs.Features().PublicLink
+	if do == nil {
+		return "", errors.New("PublicLink not supported")
+	}
+	o, err := f.NewObject(remote)
+	if err != nil {
+		// assume it is a directory
+		return do(f.cipher.EncryptDirName(remote))
+	}
+	return do(o.(*Object).Object.Remote())
+}
+
+// ChangeNotify calls the passed function with a path
+// that has had changes. If the implementation
+// uses polling, it should adhere to the given interval.
+func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+	do := f.Fs.Features().ChangeNotify
+	if do == nil {
+		return
+	}
+	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		var (
+			err       error
+			decrypted string
+		)
+		switch entryType {
+		case fs.EntryDirectory:
+			decrypted, err = f.cipher.DecryptDirName(path)
+		case fs.EntryObject:
+			decrypted, err = f.cipher.DecryptFileName(path)
+		default:
+			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
+			return
+		}
+		if err != nil {
+			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+			return
+		}
+		notifyFunc(decrypted, entryType)
+	}
+	do(wrappedNotifyFunc, pollIntervalChan)
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -774,6 +841,34 @@ func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	do, ok := o.Object.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// SetTier performs changing storage tier of the Object if
+// multiple storage classes supported
+func (o *Object) SetTier(tier string) error {
+	do, ok := o.Object.(fs.SetTierer)
+	if !ok {
+		return errors.New("crypt: underlying remote does not support SetTier")
+	}
+	return do.SetTier(tier)
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *Object) GetTier() string {
+	do, ok := o.Object.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -787,7 +882,15 @@ var (
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.Wrapper         = (*Fs)(nil)
+	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+	_ fs.SetTierer       = (*Object)(nil)
+	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_test.go

@@ -21,8 +21,10 @@ func TestIntegration(t *testing.T) {
 		t.Skip("Skipping as -remote not set")
 	}
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: *fstest.RemoteName,
-		NilObject:  (*crypt.Object)(nil),
+		RemoteName:                   *fstest.RemoteName,
+		NilObject:                    (*crypt.Object)(nil),
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -42,6 +44,8 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -61,6 +65,8 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -80,6 +86,8 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters: true,
+		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }

backend/drive/drive.go

@@ -1,7 +1,4 @@
 // Package drive interfaces with the Google Drive object storage system
-
-// +build go1.9
-
 package drive
 
 // FIXME need to deal with some corner cases
@@ -186,10 +183,10 @@ func init() {
 		},
 		Options: []fs.Option{{
 			Name: config.ConfigClientID,
-			Help: "Google Application Client Id\nLeave blank normally.",
+			Help: "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance.",
 		}, {
 			Name: config.ConfigClientSecret,
-			Help: "Google Application Client Secret\nLeave blank normally.",
+			Help: "Google Application Client Secret\nSetting your own is recommended.",
 		}, {
 			Name: "scope",
 			Help: "Scope that rclone should use when requesting access from drive.",
@@ -240,6 +237,22 @@ func init() {
 			Default:  false,
 			Help:     "Skip google documents in all listings.\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
+		}, {
+			Name:    "skip_checksum_gphotos",
+			Default: false,
+			Help: `Skip MD5 checksum on Google photos and videos only.
+
+Use this if you get checksum errors when transferring Google photos or
+videos.
+
+Setting this flag will cause Google photos and videos to return a
+blank MD5 checksum.
+
+Google photos are identifed by being in the "photos" space.
+
+Corrupted checksums are caused by Google modifying the image/video but
+not updating the checksum.`,
+			Advanced: true,
 		}, {
 			Name:    "shared_with_me",
 			Default: false,
@@ -354,6 +367,14 @@ will download it anyway.`,
 			Default:  false,
 			Help:     "Keep new head revision of each file forever.",
 			Advanced: true,
+		}, {
+			Name:    "size_as_quota",
+			Default: false,
+			Help: `Show storage quota usage for file size.
+
+The storage used by a file is the size of the current version plus any
+older versions that have been set to keep forever.`,
+			Advanced: true,
 		}, {
 			Name:     "v2_download_min_size",
 			Default:  fs.SizeSuffix(-1),
@@ -369,6 +390,16 @@ will download it anyway.`,
 			Default:  defaultBurst,
 			Help:     "Number of API calls to allow without sleeping.",
 			Advanced: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different drive configs.
+
+This can be useful if you wish to do a server side copy between two
+different Google drives.  Note that this isn't enabled by default
+because it isn't easy to tell if it will work beween any two
+configurations.`,
+			Advanced: true,
 		}},
 	})
 
@@ -396,6 +427,7 @@ type Options struct {
 	AuthOwnerOnly             bool          `config:"auth_owner_only"`
 	UseTrash                  bool          `config:"use_trash"`
 	SkipGdocs                 bool          `config:"skip_gdocs"`
+	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool          `config:"shared_with_me"`
 	TrashedOnly               bool          `config:"trashed_only"`
 	Extensions                string        `config:"formats"`
@@ -410,9 +442,11 @@ type Options struct {
 	ChunkSize                 fs.SizeSuffix `config:"chunk_size"`
 	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
 	KeepRevisionForever       bool          `config:"keep_revision_forever"`
+	SizeAsQuota               bool          `config:"size_as_quota"`
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
 	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
 	PacerBurst                int           `config:"pacer_burst"`
+	ServerSideAcrossConfigs   bool          `config:"server_side_across_configs"`
 }
 
 // Fs represents a remote drive server
@@ -426,7 +460,7 @@ type Fs struct {
 	client           *http.Client       // authorized client
 	rootFolderID     string             // the id of the root folder
 	dirCache         *dircache.DirCache // Map of directory path to directory id
-	pacer            *pacer.Pacer       // To pace the API calls
+	pacer            *fs.Pacer          // To pace the API calls
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
@@ -615,6 +649,12 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, inc
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
+	if f.opt.SkipChecksumGphotos {
+		fields += ",spaces"
+	}
+	if f.opt.SizeAsQuota {
+		fields += ",quotaBytesUsed"
+	}
 
 	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
 
@@ -676,28 +716,33 @@ func isPowerOfTwo(x int64) bool {
 }
 
 // add a charset parameter to all text/* MIME types
-func fixMimeType(mimeType string) string {
-	mediaType, param, err := mime.ParseMediaType(mimeType)
+func fixMimeType(mimeTypeIn string) string {
+	if mimeTypeIn == "" {
+		return ""
+	}
+	mediaType, param, err := mime.ParseMediaType(mimeTypeIn)
 	if err != nil {
-		return mimeType
+		return mimeTypeIn
 	}
-	if strings.HasPrefix(mimeType, "text/") && param["charset"] == "" {
+	mimeTypeOut := mimeTypeIn
+	if strings.HasPrefix(mediaType, "text/") && param["charset"] == "" {
 		param["charset"] = "utf-8"
-		mimeType = mime.FormatMediaType(mediaType, param)
+		mimeTypeOut = mime.FormatMediaType(mediaType, param)
 	}
-	return mimeType
+	if mimeTypeOut == "" {
+		panic(errors.Errorf("unable to fix MIME type %q", mimeTypeIn))
+	}
+	return mimeTypeOut
 }
-func fixMimeTypeMap(m map[string][]string) map[string][]string {
-	for _, v := range m {
+func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
+	out = make(map[string][]string, len(in))
+	for k, v := range in {
 		for i, mt := range v {
-			fixed := fixMimeType(mt)
-			if fixed == "" {
-				panic(errors.Errorf("unable to fix MIME type %q", mt))
-			}
-			v[i] = fixed
+			v[i] = fixMimeType(mt)
 		}
+		out[fixMimeType(k)] = v
 	}
-	return m
+	return out
 }
 func isInternalMimeType(mimeType string) bool {
 	return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
@@ -789,8 +834,8 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer(opt *Options) *pacer.Pacer {
-	return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
+func newPacer(opt *Options) *fs.Pacer {
+	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
@@ -902,6 +947,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f)
 
 	// Create a new authorized Drive client.
@@ -984,18 +1030,31 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 	if f.opt.UseCreatedDate {
 		modifiedDate = info.CreatedTime
 	}
+	size := info.Size
+	if f.opt.SizeAsQuota {
+		size = info.QuotaBytesUsed
+	}
 	return baseObject{
 		fs:           f,
 		remote:       remote,
 		id:           info.Id,
 		modifiedDate: modifiedDate,
 		mimeType:     info.MimeType,
-		bytes:        info.Size,
+		bytes:        size,
 	}
 }
 
 // newRegularObject creates a fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
+	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
+	if f.opt.SkipChecksumGphotos {
+		for _, space := range info.Spaces {
+			if space == "photos" {
+				info.Md5Checksum = ""
+				break
+			}
+		}
+	}
 	return &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, info.Id),
@@ -1837,16 +1896,24 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		remote = remote[:len(remote)-len(ext)]
 	}
 
+	// Look to see if there is an existing object
+	existingObject, _ := f.NewObject(remote)
+
 	createInfo, err := f.createFileInfo(remote, src.ModTime())
 	if err != nil {
 		return nil, err
 	}
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Copy(srcObj.id, createInfo).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
 			Do()
 		return shouldRetry(err)
@@ -1854,7 +1921,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.newObjectWithInfo(remote, info)
+	newObject, err := f.newObjectWithInfo(remote, info)
+	if err != nil {
+		return nil, err
+	}
+	if existingObject != nil {
+		err = existingObject.Remove()
+		if err != nil {
+			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
+		}
+	}
+	return newObject, nil
 }
 
 // Purge deletes all the files and the container
@@ -1980,6 +2057,11 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
@@ -1987,7 +2069,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			Do()
 		return shouldRetry(err)
 	})
@@ -1998,6 +2080,20 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, info)
 }
 
+// ShouldSupportTeamDrives returns the request should support TeamDrives
+func (f *Fs) ShouldSupportTeamDrives(src fs.Object) (bool, error) {
+	srcIsTeamDrive := false
+	if srcFs, ok := src.Fs().(*Fs); ok {
+		srcIsTeamDrive = srcFs.isTeamDrive
+	}
+
+	if f.isTeamDrive {
+		return true, nil
+	}
+
+	return srcIsTeamDrive, nil
+}
+
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(remote string) (link string, err error) {
 	id, err := f.dirCache.FindDir(remote, false)
@@ -2430,6 +2526,10 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
 		return req, nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
+	if o.bytes == 0 {
+		// Don't supply range requests for 0 length objects as they always fail
+		delete(req.Header, "Range")
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
 		if err == nil {

backend/drive/drive_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_test.go

@@ -1,7 +1,5 @@
 // Test Drive filesystem interface
 
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package drive

backend/drive/upload.go

@@ -8,8 +8,6 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
-// +build go1.9
-
 package drive
 
 import (

backend/dropbox/dropbox.go

@@ -160,7 +160,7 @@ type Fs struct {
 	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *pacer.Pacer   // To pace the API calls
+	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
 }
 
@@ -209,7 +209,7 @@ func shouldRetry(err error) (bool, error) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
 			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
-			time.Sleep(time.Duration(e.RateLimitError.RetryAfter) * time.Second)
+			err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
 		}
 		return true, err
 	}
@@ -273,7 +273,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	config := dropbox.Config{
 		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo

backend/ftp/ftp.go

@@ -2,6 +2,7 @@
 package ftp
 
 import (
+	"crypto/tls"
 	"io"
 	"net/textproto"
 	"os"
@@ -15,6 +16,7 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -45,6 +47,20 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
+			}, {
+				Name:    "tls",
+				Help:    "Use FTP over TLS (Implicit)",
+				Default: false,
+			}, {
+				Name:     "concurrency",
+				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
+				Default:  0,
+				Advanced: true,
+			}, {
+				Name:     "no_check_certificate",
+				Help:     "Do not verify the TLS certificate of the server",
+				Default:  false,
+				Advanced: true,
 			},
 		},
 	})
@@ -52,10 +68,13 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host string `config:"host"`
-	User string `config:"user"`
-	Pass string `config:"pass"`
-	Port string `config:"port"`
+	Host              string `config:"host"`
+	User              string `config:"user"`
+	Pass              string `config:"pass"`
+	Port              string `config:"port"`
+	TLS               bool   `config:"tls"`
+	Concurrency       int    `config:"concurrency"`
+	SkipVerifyTLSCert bool   `config:"no_check_certificate"`
 }
 
 // Fs represents a remote FTP server
@@ -70,6 +89,7 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
+	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -112,7 +132,15 @@ func (f *Fs) Features() *fs.Features {
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
-	c, err := ftp.DialTimeout(f.dialAddr, fs.Config.ConnectTimeout)
+	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
+	if f.opt.TLS {
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+		}
+		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
+	}
+	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
 		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
 		return nil, errors.Wrap(err, "ftpConnection Dial")
@@ -128,6 +156,9 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		f.tokens.Get()
+	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -147,6 +178,9 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		defer f.tokens.Put()
+	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -189,7 +223,11 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	}
 
 	dialAddr := opt.Host + ":" + port
-	u := "ftp://" + path.Join(dialAddr+"/", root)
+	protocol := "ftp://"
+	if opt.TLS {
+		protocol = "ftps://"
+	}
+	u := protocol + path.Join(dialAddr+"/", root)
 	f := &Fs{
 		name:     name,
 		root:     root,
@@ -198,6 +236,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
+		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -330,11 +369,36 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
 	}
-	files, err := c.List(path.Join(f.root, dir))
-	f.putFtpConnection(&c, err)
-	if err != nil {
-		return nil, translateErrorDir(err)
+
+	var listErr error
+	var files []*ftp.Entry
+
+	resultchan := make(chan []*ftp.Entry, 1)
+	errchan := make(chan error, 1)
+	go func() {
+		result, err := c.List(path.Join(f.root, dir))
+		f.putFtpConnection(&c, err)
+		if err != nil {
+			errchan <- err
+			return
+		}
+		resultchan <- result
+	}()
+
+	// Wait for List for up to Timeout seconds
+	timer := time.NewTimer(fs.Config.Timeout)
+	select {
+	case listErr = <-errchan:
+		timer.Stop()
+		return nil, translateErrorDir(listErr)
+	case files = <-resultchan:
+		timer.Stop()
+	case <-timer.C:
+		// if timer fired assume no error but connection dead
+		fs.Errorf(f, "Timeout when waiting for List")
+		return nil, errors.New("Timeout when waiting for List")
 	}
+
 	// Annoyingly FTP returns success for a directory which
 	// doesn't exist, so check it really doesn't exist if no
 	// entries found.

backend/googlecloudstorage/googlecloudstorage.go

@@ -1,7 +1,4 @@
 // Package googlecloudstorage provides an interface to Google Cloud Storage
-
-// +build go1.9
-
 package googlecloudstorage
 
 /*
@@ -16,6 +13,7 @@ FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 erro
 */
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -45,6 +43,8 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
+
+	// NOTE: This API is deprecated
 	storage "google.golang.org/api/storage/v1"
 )
 
@@ -144,6 +144,22 @@ func init() {
 				Value: "publicReadWrite",
 				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",
 			}},
+		}, {
+			Name: "bucket_policy_only",
+			Help: `Access checks should use bucket-level IAM policies.
+
+If you want to upload objects to a bucket with Bucket Policy Only set
+then you will need to set this.
+
+When it is set, rclone:
+
+- ignores ACLs set on buckets
+- ignores ACLs set on objects
+- creates buckets with Bucket Policy Only set
+
+Docs: https://cloud.google.com/storage/docs/bucket-policy-only
+`,
+			Default: false,
 		}, {
 			Name: "location",
 			Help: "Location for the newly created buckets.",
@@ -241,6 +257,7 @@ type Options struct {
 	ServiceAccountCredentials string `config:"service_account_credentials"`
 	ObjectACL                 string `config:"object_acl"`
 	BucketACL                 string `config:"bucket_acl"`
+	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
 	Location                  string `config:"location"`
 	StorageClass              string `config:"storage_class"`
 }
@@ -256,7 +273,7 @@ type Fs struct {
 	bucket     string           // the bucket we are working on
 	bucketOKMu sync.Mutex       // mutex to protect bucket OK
 	bucketOK   bool             // true if we have created the bucket
-	pacer      *pacer.Pacer     // To pace the API calls
+	pacer      *fs.Pacer        // To pace the API calls
 }
 
 // Object describes a storage object
@@ -381,7 +398,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
+			ctx := context.Background()
+			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
+			}
 		}
 	}
 
@@ -395,7 +416,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		root:   directory,
 		opt:    *opt,
-		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
+		pacer:  fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -709,8 +730,19 @@ func (f *Fs) Mkdir(dir string) (err error) {
 		Location:     f.opt.Location,
 		StorageClass: f.opt.StorageClass,
 	}
+	if f.opt.BucketPolicyOnly {
+		bucket.IamConfiguration = &storage.BucketIamConfiguration{
+			BucketPolicyOnly: &storage.BucketIamConfigurationBucketPolicyOnly{
+				Enabled: true,
+			},
+		}
+	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
+		insertBucket := f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket)
+		if !f.opt.BucketPolicyOnly {
+			insertBucket.PredefinedAcl(f.opt.BucketACL)
+		}
+		_, err = insertBucket.Do()
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -971,12 +1003,15 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		Bucket:      o.fs.bucket,
 		Name:        o.fs.root + o.remote,
 		ContentType: fs.MimeType(src),
-		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
+		insertObject := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
+		if !o.fs.opt.BucketPolicyOnly {
+			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
+		}
+		newObject, err = insertObject.Do()
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,7 +1,5 @@
 // Test GoogleCloudStorage filesystem interface
 
-// +build go1.9
-
 package googlecloudstorage_test
 
 import (

backend/googlecloudstorage/googlecloudstorage_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package googlecloudstorage

backend/http/http.go

@@ -6,6 +6,7 @@ package http
 
 import (
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -44,6 +45,22 @@ func init() {
 				Value: "https://user:pass@example.com",
 				Help:  "Connect to example.com using a username and password",
 			}},
+		}, {
+			Name: "no_slash",
+			Help: `Set this if the site doesn't end directories with /
+
+Use this if your target website does not use / on the end of
+directories.
+
+A / on the end of a path is how rclone normally tells the difference
+between files and directories.  If this flag is set, then rclone will
+treat all files with Content-Type: text/html as directories and read
+URLs from them rather than downloading them.
+
+Note that this may cause rclone to confuse genuine HTML files with
+directories.`,
+			Default:  false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -52,6 +69,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
+	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -270,14 +288,20 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var walk func(*html.Node)
+	var (
+		walk func(*html.Node)
+		seen = make(map[string]struct{})
+	)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						names = append(names, name)
+						if _, found := seen[name]; !found {
+							names = append(names, name)
+							seen[name] = struct{}{}
+						}
 					}
 					break
 				}
@@ -302,14 +326,16 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil && res.StatusCode == http.StatusNotFound {
-		return nil, fs.ErrorDirNotFound
+	if err == nil {
+		defer fs.CheckClose(res.Body, &err)
+		if res.StatusCode == http.StatusNotFound {
+			return nil, fs.ErrorDirNotFound
+		}
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
-	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -353,11 +379,16 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			if err = file.stat(); err != nil {
+			switch err = file.stat(); err {
+			case nil:
+				entries = append(entries, file)
+			case fs.ErrorNotAFile:
+				// ...found a directory not a file
+				dir := fs.NewDir(remote, timeUnset)
+				entries = append(entries, dir)
+			default:
 				fs.Debugf(remote, "skipping because of error: %v", err)
-				continue
 			}
-			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -433,6 +464,16 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	// If NoSlash is set then check ContentType to see if it is a directory
+	if o.fs.opt.NoSlash {
+		mediaType, _, err := mime.ParseMediaType(o.contentType)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
+		}
+		if mediaType == "text/html" {
+			return fs.ErrorNotAFile
+		}
+	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.8
-
 package http
 
 import (
@@ -65,7 +63,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs) {
+func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -93,15 +91,29 @@ func testListRoot(t *testing.T, f fs.Fs) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	assert.Equal(t, int64(7), e.Size())
-	_, ok = e.(*Object)
-	assert.True(t, ok)
+	if noSlash {
+		assert.Equal(t, int64(-1), e.Size())
+		_, ok = e.(fs.Directory)
+		assert.True(t, ok)
+	} else {
+		assert.Equal(t, int64(41), e.Size())
+		_, ok = e.(*Object)
+		assert.True(t, ok)
+	}
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f)
+	testListRoot(t, f, false)
+}
+
+func TestListRootNoSlash(t *testing.T) {
+	f, tidy := prepare(t)
+	f.(*Fs).opt.NoSlash = true
+	defer tidy()
+
+	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -194,7 +206,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f)
+	testListRoot(t, f, false)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-potato
+<a href="two.html/file.txt">file.txt</a>

backend/hubic/hubic_test.go

@@ -11,7 +11,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestHubic:",
-		NilObject:  (*hubic.Object)(nil),
+		RemoteName:          "TestHubic:",
+		NilObject:           (*hubic.Object)(nil),
+		SkipFsCheckWrap:     true,
+		SkipObjectCheckWrap: true,
 	})
 }

backend/jottacloud/api/types.go

@@ -314,3 +314,9 @@ type UploadResponse struct {
 	Deleted   interface{} `json:"deleted"`
 	Mime      string      `json:"mime"`
 }
+
+// DeviceRegistrationResponse is the response to registering a device
+type DeviceRegistrationResponse struct {
+	ClientID     string `json:"client_id"`
+	ClientSecret string `json:"client_secret"`
+}

backend/jottacloud/jottacloud.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -40,15 +41,21 @@ const (
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Sync" // nolint
+	defaultMountpoint           = "Archive"
 	rootURL                     = "https://www.jottacloud.com/jfs/"
 	apiURL                      = "https://api.jottacloud.com/files/v1/"
 	baseURL                     = "https://www.jottacloud.com/"
 	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
+	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
 	cachePrefix                 = "rclone-jcmd5-"
 	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
 	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
 	configUsername              = "user"
+	configClientID              = "client_id"
+	configClientSecret          = "client_secret"
+	configDevice                = "device"
+	configMountpoint            = "mountpoint"
+	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 )
 
 var (
@@ -58,14 +65,13 @@ var (
 			AuthURL:  tokenURL,
 			TokenURL: tokenURL,
 		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
+		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
 )
 
 // Register with Fs
 func init() {
+	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
 		Description: "JottaCloud",
@@ -79,14 +85,62 @@ func init() {
 				}
 			}
 
+			srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+			// ask if we should create a device specifc token: https://github.com/ncw/rclone/issues/2995
+			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+			if config.Confirm() {
+				// random generator to generate random device names
+				seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+				randonDeviceNamePartLength := 21
+				randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
+				for i := range randomDeviceNamePart {
+					randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
+				}
+				randomDeviceName := "rclone-" + string(randomDeviceNamePart)
+				fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
+
+				values := url.Values{}
+				values.Set("device_id", randomDeviceName)
+
+				// all information comes from https://github.com/ttyridal/aiojotta/wiki/Jotta-protocol-3.-Authentication#token-authentication
+				opts := rest.Opts{
+					Method:       "POST",
+					RootURL:      registerURL,
+					ContentType:  "application/x-www-form-urlencoded",
+					ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
+					Parameters:   values,
+				}
+
+				var deviceRegistration api.DeviceRegistrationResponse
+				_, err := srv.CallJSON(&opts, nil, &deviceRegistration)
+				if err != nil {
+					log.Fatalf("Failed to register device: %v", err)
+				}
+
+				m.Set(configClientID, deviceRegistration.ClientID)
+				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+			}
+
+			clientID, ok := m.Get(configClientID)
+			if !ok {
+				clientID = rcloneClientID
+			}
+			clientSecret, ok := m.Get(configClientSecret)
+			if !ok {
+				clientSecret = rcloneEncryptedClientSecret
+			}
+			oauthConfig.ClientID = clientID
+			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
 			username, ok := m.Get(configUsername)
 			if !ok {
 				log.Fatalf("No username defined")
 			}
-			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
+			password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
 
 			// prepare out token request with username and password
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
 			values := url.Values{}
 			values.Set("grant_type", "PASSWORD")
 			values.Set("password", password)
@@ -106,7 +160,7 @@ func init() {
 				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
 				if resp != nil {
 					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
+						fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
 						fmt.Printf("Enter verification code> ")
 						authCode := config.ReadLine()
 						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
@@ -129,23 +183,49 @@ func init() {
 			// finally save them in the config
 			err = oauthutil.PutToken(name, m, &token, true)
 			if err != nil {
-				log.Fatalf("Error while setting token: %s", err)
+				log.Fatalf("Error while saving token: %s", err)
+			}
+
+			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+			if config.Confirm() {
+				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+				if err != nil {
+					log.Fatalf("Failed to load oAuthClient: %s", err)
+				}
+
+				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+
+				acc, err := getAccountInfo(srv, username)
+				if err != nil {
+					log.Fatalf("Error getting devices: %s", err)
+				}
+				fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
+				var deviceNames []string
+				for i := range acc.Devices {
+					deviceNames = append(deviceNames, acc.Devices[i].Name)
+				}
+				result := config.Choose("Devices", deviceNames, nil, false)
+				m.Set(configDevice, result)
+
+				dev, err := getDeviceInfo(srv, path.Join(username, result))
+				if err != nil {
+					log.Fatalf("Error getting Mountpoint: %s", err)
+				}
+				if len(dev.MountPoints) == 0 {
+					log.Fatalf("No Mountpoints found for this device.")
+				}
+				fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
+				var mountpointNames []string
+				for i := range dev.MountPoints {
+					mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
+				}
+				result = config.Choose("Mountpoints", mountpointNames, nil, false)
+				m.Set(configMountpoint, result)
 			}
 		},
 		Options: []fs.Option{{
 			Name: configUsername,
 			Help: "User Name:",
-		}, {
-			Name:     "mountpoint",
-			Help:     "The mountpoint to use.",
-			Required: true,
-			Examples: []fs.OptionExample{{
-				Value: "Sync",
-				Help:  "Will be synced by the official client.",
-			}, {
-				Value: "Archive",
-				Help:  "Archive",
-			}},
 		}, {
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -173,6 +253,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
+	Device             string        `config:"device"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
@@ -190,7 +271,7 @@ type Fs struct {
 	endpointURL  string
 	srv          *rest.Client
 	apiSrv       *rest.Client
-	pacer        *pacer.Pacer
+	pacer        *fs.Pacer
 	tokenRenewer *oauthutil.Renew // renew the token on expiry
 }
 
@@ -280,18 +361,31 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
 	return &result, nil
 }
 
-// getAccountInfo retrieves account information
-func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
+// getAccountInfo queries general information about the account.
+// Takes rest.Client and username as parameter to be easily usable
+// during config
+func getAccountInfo(srv *rest.Client, username string) (info *api.AccountInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
-		Path:   urlPathEscape(f.user),
+		Path:   urlPathEscape(username),
 	}
 
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &info)
-		return shouldRetry(resp, err)
-	})
+	_, err = srv.CallXML(&opts, nil, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// getDeviceInfo queries Information about a jottacloud device
+func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   urlPathEscape(path),
+	}
+
+	_, err = srv.CallXML(&opts, nil, &info)
 	if err != nil {
 		return nil, err
 	}
@@ -300,12 +394,18 @@ func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
 }
 
 // setEndpointUrl reads the account id and generates the API endpoint URL
-func (f *Fs) setEndpointURL(mountpoint string) (err error) {
-	info, err := f.getAccountInfo()
+func (f *Fs) setEndpointURL() (err error) {
+	info, err := getAccountInfo(f.srv, f.user)
 	if err != nil {
 		return errors.Wrap(err, "failed to get endpoint url")
 	}
-	f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
+	if f.opt.Device == "" {
+		f.opt.Device = defaultDevice
+	}
+	if f.opt.Mountpoint == "" {
+		f.opt.Mountpoint = defaultMountpoint
+	}
+	f.endpointURL = urlPathEscape(path.Join(info.Username, f.opt.Device, f.opt.Mountpoint))
 	return nil
 }
 
@@ -381,6 +481,17 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = rcloneClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = rcloneEncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
 	// the oauth client for the api servers needs
 	// a filter to fix the grant_type issues (see above)
 	baseClient := fshttp.NewClient(fs.Config)
@@ -403,7 +514,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		opt:    *opt,
 		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
 		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
-		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:  fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -419,7 +530,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return err
 	})
 
-	err = f.setEndpointURL(opt.Mountpoint)
+	err = f.setEndpointURL()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
 	}
@@ -677,6 +788,9 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	if f.opt.Device != "Jotta" {
+		return nil, errors.New("upload not supported for devices other than Jotta")
+	}
 	o := f.createObject(src.Remote(), src.ModTime(), src.Size())
 	return o, o.Update(in, src, options...)
 }
@@ -940,7 +1054,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
-	info, err := f.getAccountInfo()
+	info, err := getAccountInfo(f.srv, f.user)
 	if err != nil {
 		return nil, err
 	}

backend/koofr/koofr.go

@@ -0,0 +1,589 @@
+package koofr
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/hash"
+
+	httpclient "github.com/koofr/go-httpclient"
+	koofrclient "github.com/koofr/go-koofrclient"
+)
+
+// Register Fs with rclone
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "koofr",
+		Description: "Koofr",
+		NewFs:       NewFs,
+		Options: []fs.Option{
+			{
+				Name:     "endpoint",
+				Help:     "The Koofr API endpoint to use",
+				Default:  "https://app.koofr.net",
+				Required: true,
+				Advanced: true,
+			}, {
+				Name:     "mountid",
+				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
+				Required: false,
+				Default:  "",
+				Advanced: true,
+			}, {
+				Name:     "user",
+				Help:     "Your Koofr user name",
+				Required: true,
+			}, {
+				Name:       "password",
+				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
+				IsPassword: true,
+				Required:   true,
+			},
+		},
+	})
+}
+
+// Options represent the configuration of the Koofr backend
+type Options struct {
+	Endpoint string `config:"endpoint"`
+	MountID  string `config:"mountid"`
+	User     string `config:"user"`
+	Password string `config:"password"`
+}
+
+// A Fs is a representation of a remote Koofr Fs
+type Fs struct {
+	name     string
+	mountID  string
+	root     string
+	opt      Options
+	features *fs.Features
+	client   *koofrclient.KoofrClient
+}
+
+// An Object on the remote Koofr Fs
+type Object struct {
+	fs     *Fs
+	remote string
+	info   koofrclient.FileInfo
+}
+
+func base(pth string) string {
+	rv := path.Base(pth)
+	if rv == "" || rv == "." {
+		rv = "/"
+	}
+	return rv
+}
+
+func dir(pth string) string {
+	rv := path.Dir(pth)
+	if rv == "" || rv == "." {
+		rv = "/"
+	}
+	return rv
+}
+
+// String returns a string representation of the remote Object
+func (o *Object) String() string {
+	return o.remote
+}
+
+// Remote returns the remote path of the Object, relative to Fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the Object
+func (o *Object) ModTime() time.Time {
+	return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000)
+}
+
+// Size return the size of the Object in bytes
+func (o *Object) Size() int64 {
+	return o.info.Size
+}
+
+// Fs returns a reference to the Koofr Fs containing the Object
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns an MD5 hash of the Object
+func (o *Object) Hash(typ hash.Type) (string, error) {
+	if typ == hash.MD5 {
+		return o.info.Hash, nil
+	}
+	return "", nil
+}
+
+// fullPath returns full path of the remote Object (including Fs root)
+func (o *Object) fullPath() string {
+	return o.fs.fullPath(o.remote)
+}
+
+// Storable returns true if the Object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime is not supported
+func (o *Object) SetModTime(mtime time.Time) error {
+	return nil
+}
+
+// Open opens the Object for reading
+func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+	var sOff, eOff int64 = 0, -1
+
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			sOff = x.Offset
+		case *fs.RangeOption:
+			sOff = x.Start
+			eOff = x.End
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	if sOff == 0 && eOff < 0 {
+		return o.fs.client.FilesGet(o.fs.mountID, o.fullPath())
+	}
+	if sOff < 0 {
+		sOff = o.Size() - eOff
+		eOff = o.Size()
+	}
+	if eOff > o.Size() {
+		eOff = o.Size()
+	}
+	span := &koofrclient.FileSpan{
+		Start: sOff,
+		End:   eOff,
+	}
+	return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span)
+}
+
+// Update updates the Object contents
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	putopts := &koofrclient.PutFilter{
+		ForceOverwrite:    true,
+		NoRename:          true,
+		IgnoreNonExisting: true,
+	}
+	fullPath := o.fullPath()
+	dirPath := dir(fullPath)
+	name := base(fullPath)
+	err := o.fs.mkdir(dirPath)
+	if err != nil {
+		return err
+	}
+	info, err := o.fs.client.FilesPutOptions(o.fs.mountID, dirPath, name, in, putopts)
+	if err != nil {
+		return err
+	}
+	o.info = *info
+	return nil
+}
+
+// Remove deletes the remote Object
+func (o *Object) Remove() error {
+	return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath())
+}
+
+// Name returns the name of the Fs
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root path of the Fs
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a string representation of the Fs
+func (f *Fs) String() string {
+	return "koofr:" + f.mountID + ":" + f.root
+}
+
+// Features returns the optional features supported by this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision denotes that setting modification times is not supported
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns a set of hashes are Provided by the Fs
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.MD5)
+}
+
+// fullPath constructs a full, absolute path from a Fs root relative path,
+func (f *Fs) fullPath(part string) string {
+	return path.Join("/", f.root, part)
+}
+
+// NewFs constructs a new filesystem given a root path and configuration options
+func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	pass, err := obscure.Reveal(opt.Password)
+	if err != nil {
+		return nil, err
+	}
+	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
+	basicAuth := fmt.Sprintf("Basic %s",
+		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
+	client.HTTPClient.Headers.Set("Authorization", basicAuth)
+	mounts, err := client.Mounts()
+	if err != nil {
+		return nil, err
+	}
+	f := &Fs{
+		name:   name,
+		root:   root,
+		opt:    *opt,
+		client: client,
+	}
+	f.features = (&fs.Features{
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		BucketBased:             false,
+		CanHaveEmptyDirectories: true,
+	}).Fill(f)
+	for _, m := range mounts {
+		if opt.MountID != "" {
+			if m.Id == opt.MountID {
+				f.mountID = m.Id
+				break
+			}
+		} else if m.IsPrimary {
+			f.mountID = m.Id
+			break
+		}
+	}
+	if f.mountID == "" {
+		if opt.MountID == "" {
+			return nil, errors.New("Failed to find primary mount")
+		}
+		return nil, errors.New("Failed to find mount " + opt.MountID)
+	}
+	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
+	if err == nil && rootFile.Type != "dir" {
+		f.root = dir(f.root)
+		err = fs.ErrorIsFile
+	} else {
+		err = nil
+	}
+	return f, err
+}
+
+// List returns a list of items in a directory
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return nil, translateErrorsDir(err)
+	}
+	entries = make([]fs.DirEntry, len(files))
+	for i, file := range files {
+		if file.Type == "dir" {
+			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
+		} else {
+			entries[i] = &Object{
+				fs:     f,
+				info:   file,
+				remote: path.Join(dir, file.Name),
+			}
+		}
+	}
+	return entries, nil
+}
+
+// NewObject creates a new remote Object for a given remote path
+func (f *Fs) NewObject(remote string) (obj fs.Object, err error) {
+	info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote))
+	if err != nil {
+		return nil, translateErrorsObject(err)
+	}
+	if info.Type == "dir" {
+		return nil, fs.ErrorNotAFile
+	}
+	return &Object{
+		fs:     f,
+		info:   info,
+		remote: remote,
+	}, nil
+}
+
+// Put updates a remote Object
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
+	putopts := &koofrclient.PutFilter{
+		ForceOverwrite:    true,
+		NoRename:          true,
+		IgnoreNonExisting: true,
+	}
+	fullPath := f.fullPath(src.Remote())
+	dirPath := dir(fullPath)
+	name := base(fullPath)
+	err = f.mkdir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	info, err := f.client.FilesPutOptions(f.mountID, dirPath, name, in, putopts)
+	if err != nil {
+		return nil, translateErrorsObject(err)
+	}
+	return &Object{
+		fs:     f,
+		info:   *info,
+		remote: src.Remote(),
+	}, nil
+}
+
+// PutStream updates a remote Object with a stream of unknown size
+func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(in, src, options...)
+}
+
+// isBadRequest is a predicate which holds true iff the error returned was
+// HTTP status 400
+func isBadRequest(err error) bool {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusBadRequest {
+			return true
+		}
+	}
+	return false
+}
+
+// translateErrorsDir translates koofr errors to rclone errors (for a dir
+// operation)
+func translateErrorsDir(err error) error {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusNotFound {
+			return fs.ErrorDirNotFound
+		}
+	}
+	return err
+}
+
+// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
+func translateErrorsObject(err error) error {
+	switch err := err.(type) {
+	case httpclient.InvalidStatusError:
+		if err.Got == http.StatusNotFound {
+			return fs.ErrorObjectNotFound
+		}
+	}
+	return err
+}
+
+// mkdir creates a directory at the given remote path. Creates ancestors if
+// neccessary
+func (f *Fs) mkdir(fullPath string) error {
+	if fullPath == "/" {
+		return nil
+	}
+	info, err := f.client.FilesInfo(f.mountID, fullPath)
+	if err == nil && info.Type == "dir" {
+		return nil
+	}
+	err = translateErrorsDir(err)
+	if err != nil && err != fs.ErrorDirNotFound {
+		return err
+	}
+	dirs := strings.Split(fullPath, "/")
+	parent := "/"
+	for _, part := range dirs {
+		if part == "" {
+			continue
+		}
+		info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part))
+		if err != nil || info.Type != "dir" {
+			err = translateErrorsDir(err)
+			if err != nil && err != fs.ErrorDirNotFound {
+				return err
+			}
+			err = f.client.FilesNewFolder(f.mountID, parent, part)
+			if err != nil && !isBadRequest(err) {
+				return err
+			}
+		}
+		parent = path.Join(parent, part)
+	}
+	return nil
+}
+
+// Mkdir creates a directory at the given remote path. Creates ancestors if
+// necessary
+func (f *Fs) Mkdir(dir string) error {
+	fullPath := f.fullPath(dir)
+	return f.mkdir(fullPath)
+}
+
+// Rmdir removes an (empty) directory at the given remote path
+func (f *Fs) Rmdir(dir string) error {
+	files, err := f.client.FilesList(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return translateErrorsDir(err)
+	}
+	if len(files) > 0 {
+		return fs.ErrorDirectoryNotEmpty
+	}
+	err = f.client.FilesDelete(f.mountID, f.fullPath(dir))
+	if err != nil {
+		return translateErrorsDir(err)
+	}
+	return nil
+}
+
+// Copy copies a remote Object to the given path
+func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+	dstFullPath := f.fullPath(remote)
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return nil, fs.ErrorCantCopy
+	}
+	err = f.client.FilesCopy((src.(*Object)).fs.mountID,
+		(src.(*Object)).fs.fullPath((src.(*Object)).remote),
+		f.mountID, dstFullPath)
+	if err != nil {
+		return nil, fs.ErrorCantCopy
+	}
+	return f.NewObject(remote)
+}
+
+// Move moves a remote Object to the given path
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
+	srcObj := src.(*Object)
+	dstFullPath := f.fullPath(remote)
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return nil, fs.ErrorCantMove
+	}
+	err = f.client.FilesMove(srcObj.fs.mountID,
+		srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath)
+	if err != nil {
+		return nil, fs.ErrorCantMove
+	}
+	return f.NewObject(remote)
+}
+
+// DirMove moves a remote directory to the given path
+func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs := src.(*Fs)
+	srcFullPath := srcFs.fullPath(srcRemote)
+	dstFullPath := f.fullPath(dstRemote)
+	if srcFs.mountID == f.mountID && srcFullPath == dstFullPath {
+		return fs.ErrorDirExists
+	}
+	dstDir := dir(dstFullPath)
+	err := f.mkdir(dstDir)
+	if err != nil {
+		return fs.ErrorCantDirMove
+	}
+	err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath)
+	if err != nil {
+		return fs.ErrorCantDirMove
+	}
+	return nil
+}
+
+// About reports space usage (with a MB precision)
+func (f *Fs) About() (*fs.Usage, error) {
+	mount, err := f.client.MountsDetails(f.mountID)
+	if err != nil {
+		return nil, err
+	}
+	return &fs.Usage{
+		Total:   fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024),
+		Used:    fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024),
+		Trashed: nil,
+		Other:   nil,
+		Free:    fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024),
+		Objects: nil,
+	}, nil
+}
+
+// Purge purges the complete Fs
+func (f *Fs) Purge() error {
+	err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath("")))
+	return err
+}
+
+// linkCreate is a Koofr API request for creating a public link
+type linkCreate struct {
+	Path string `json:"path"`
+}
+
+// link is a Koofr API response to creating a public link
+type link struct {
+	ID               string `json:"id"`
+	Name             string `json:"name"`
+	Path             string `json:"path"`
+	Counter          int64  `json:"counter"`
+	URL              string `json:"url"`
+	ShortURL         string `json:"shortUrl"`
+	Hash             string `json:"hash"`
+	Host             string `json:"host"`
+	HasPassword      bool   `json:"hasPassword"`
+	Password         string `json:"password"`
+	ValidFrom        int64  `json:"validFrom"`
+	ValidTo          int64  `json:"validTo"`
+	PasswordRequired bool   `json:"passwordRequired"`
+}
+
+// createLink makes a Koofr API call to create a public link
+func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) {
+	linkCreate := linkCreate{
+		Path: path,
+	}
+	linkData := link{}
+
+	request := httpclient.RequestData{
+		Method:         "POST",
+		Path:           "/api/v2/mounts/" + mountID + "/links",
+		ExpectedStatus: []int{http.StatusOK, http.StatusCreated},
+		ReqEncoding:    httpclient.EncodingJSON,
+		ReqValue:       linkCreate,
+		RespEncoding:   httpclient.EncodingJSON,
+		RespValue:      &linkData,
+	}
+
+	_, err := c.Request(&request)
+	if err != nil {
+		return nil, err
+	}
+	return &linkData, nil
+}
+
+// PublicLink creates a public link to the remote path
+func (f *Fs) PublicLink(remote string) (string, error) {
+	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
+	if err != nil {
+		return "", translateErrorsDir(err)
+	}
+	return linkData.ShortURL, nil
+}

backend/koofr/koofr_test.go

@@ -0,0 +1,14 @@
+package koofr_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKoofr:",
+	})
+}

backend/local/local.go

@@ -28,8 +28,9 @@ import (
 )
 
 // Constants
-const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
-const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
+const devUnset = 0xdeadbeefcafebabe                                       // a device id meaning it is unset
+const linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
+const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
 
 // Register with Fs
 func init() {
@@ -327,7 +328,14 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 
 	fd, err := os.Open(fsDirPath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to open directory %q", dir)
+		isPerm := os.IsPermission(err)
+		err = errors.Wrapf(err, "failed to open directory %q", dir)
+		fs.Errorf(dir, "%v", err)
+		if isPerm {
+			accounting.Stats.Error(fserrors.NoRetryError(err))
+			err = nil // ignore error but fail sync
+		}
+		return nil, err
 	}
 	defer func() {
 		cerr := fd.Close()
@@ -337,12 +345,38 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	}()
 
 	for {
-		fis, err := fd.Readdir(1024)
-		if err == io.EOF && len(fis) == 0 {
-			break
+		var fis []os.FileInfo
+		if useReadDir {
+			// Windows and Plan9 read the directory entries with the stat information in which
+			// shouldn't fail because of unreadable entries.
+			fis, err = fd.Readdir(1024)
+			if err == io.EOF && len(fis) == 0 {
+				break
+			}
+		} else {
+			// For other OSes we read the names only (which shouldn't fail) then stat the
+			// individual ourselves so we can log errors but not fail the directory read.
+			var names []string
+			names, err = fd.Readdirnames(1024)
+			if err == io.EOF && len(names) == 0 {
+				break
+			}
+			if err == nil {
+				for _, name := range names {
+					namepath := filepath.Join(fsDirPath, name)
+					fi, fierr := os.Lstat(namepath)
+					if fierr != nil {
+						err = errors.Wrapf(err, "failed to read directory %q", namepath)
+						fs.Errorf(dir, "%v", fierr)
+						accounting.Stats.Error(fserrors.NoRetryError(fierr)) // fail the sync
+						continue
+					}
+					fis = append(fis, fi)
+				}
+			}
 		}
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to read directory %q", dir)
+			return nil, errors.Wrap(err, "failed to read directory entry")
 		}
 
 		for _, fi := range fis {
@@ -709,9 +743,10 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 
 	o.fs.objectHashesMu.Lock()
 	hashes := o.hashes
+	hashValue, hashFound := o.hashes[r]
 	o.fs.objectHashesMu.Unlock()
 
-	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
+	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -722,7 +757,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
-		hashes, err = hash.Stream(in)
+		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to read")
@@ -730,11 +765,16 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		if closeErr != nil {
 			return "", errors.Wrap(closeErr, "hash: failed to close")
 		}
+		hashValue = hashes[r]
 		o.fs.objectHashesMu.Lock()
-		o.hashes = hashes
+		if o.hashes == nil {
+			o.hashes = hashes
+		} else {
+			o.hashes[r] = hashValue
+		}
 		o.fs.objectHashesMu.Unlock()
 	}
-	return hashes[r], nil
+	return hashValue, nil
 }
 
 // Size returns the size of an object in bytes
@@ -998,6 +1038,36 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	return o.lstat()
 }
 
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(remote string, size int64) (fs.WriterAtCloser, error) {
+	// Temporary Object under construction
+	o := f.newObject(remote, "")
+
+	err := o.mkdirAll()
+	if err != nil {
+		return nil, err
+	}
+
+	if o.translatedLink {
+		return nil, errors.New("can't open a symlink for random writing")
+	}
+
+	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return nil, err
+	}
+	// Pre-allocate the file for performance reasons
+	err = preAllocate(size, out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	}
+	return out, nil
+}
+
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
 	// Don't overwrite the info if we don't need to
@@ -1139,10 +1209,11 @@ func cleanWindowsName(f *Fs, name string) string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs          = &Fs{}
-	_ fs.Purger      = &Fs{}
-	_ fs.PutStreamer = &Fs{}
-	_ fs.Mover       = &Fs{}
-	_ fs.DirMover    = &Fs{}
-	_ fs.Object      = &Object{}
+	_ fs.Fs             = &Fs{}
+	_ fs.Purger         = &Fs{}
+	_ fs.PutStreamer    = &Fs{}
+	_ fs.Mover          = &Fs{}
+	_ fs.DirMover       = &Fs{}
+	_ fs.OpenWriterAter = &Fs{}
+	_ fs.Object         = &Object{}
 )

backend/local/preallocate_unix.go

@@ -4,16 +4,40 @@ package local
 
 import (
 	"os"
+	"sync/atomic"
 
+	"github.com/ncw/rclone/fs"
 	"golang.org/x/sys/unix"
 )
 
+var (
+	fallocFlags = [...]uint32{
+		unix.FALLOC_FL_KEEP_SIZE,                             // Default
+		unix.FALLOC_FL_KEEP_SIZE | unix.FALLOC_FL_PUNCH_HOLE, // for ZFS #3066
+	}
+	fallocFlagsIndex int32
+)
+
 // preAllocate the file for performance reasons
 func preAllocate(size int64, out *os.File) error {
 	if size <= 0 {
 		return nil
 	}
-	err := unix.Fallocate(int(out.Fd()), unix.FALLOC_FL_KEEP_SIZE, 0, size)
+	index := atomic.LoadInt32(&fallocFlagsIndex)
+again:
+	if index >= int32(len(fallocFlags)) {
+		return nil // Fallocate is disabled
+	}
+	flags := fallocFlags[index]
+	err := unix.Fallocate(int(out.Fd()), flags, 0, size)
+	if err == unix.ENOTSUP {
+		// Try the next flags combination
+		index++
+		atomic.StoreInt32(&fallocFlagsIndex, index)
+		fs.Debugf(nil, "preAllocate: got error on fallocate, trying combination %d/%d: %v", index, len(fallocFlags), err)
+		goto again
+
+	}
 	// FIXME could be doing something here
 	// if err == unix.ENOSPC {
 	// 	log.Printf("No space")

backend/mega/mega.go

@@ -98,7 +98,7 @@ type Fs struct {
 	opt        Options      // parsed config options
 	features   *fs.Features // optional features
 	srv        *mega.Mega   // the connection to the server
-	pacer      *pacer.Pacer // pacer for API calls
+	pacer      *fs.Pacer    // pacer for API calls
 	rootNodeMu sync.Mutex   // mutex for _rootNode
 	_rootNode  *mega.Node   // root node - call findRoot to use this
 	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir
@@ -217,7 +217,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   srv,
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		DuplicateFiles:          true,
@@ -402,6 +402,35 @@ func (f *Fs) clearRoot() {
 	//log.Printf("cleared root directory")
 }
 
+// CleanUp deletes all files currently in trash
+func (f *Fs) CleanUp() (err error) {
+	trash := f.srv.FS.GetTrash()
+	items := []*mega.Node{}
+	_, err = f.list(trash, func(item *mega.Node) bool {
+		items = append(items, item)
+		return false
+	})
+	if err != nil {
+		return errors.Wrap(err, "CleanUp failed to list items in trash")
+	}
+	fs.Infof(f, "Deleting %d items from the trash", len(items))
+	errors := 0
+	// similar to f.deleteNode(trash) but with HardDelete as true
+	for _, item := range items {
+		fs.Debugf(f, "Deleting trash %q", item.GetName())
+		deleteErr := f.pacer.Call(func() (bool, error) {
+			err := f.srv.Delete(item, true)
+			return shouldRetry(err)
+		})
+		if deleteErr != nil {
+			err = deleteErr
+			errors++
+		}
+	}
+	fs.Infof(f, "Deleted %d items from the trash with %d errors", len(items), errors)
+	return err
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.

backend/onedrive/onedrive.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/lib/atexit"
+
 	"github.com/ncw/rclone/backend/onedrive/api"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -261,7 +263,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the one drive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
@@ -335,8 +337,13 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
 // if `relPath` == "", it reads the metadata for the item with that ID.
+//
+// We address items using the pattern `drives/driveID/items/itemID:/relativePath`
+// instead of simply using `drives/driveID/root:/itemPath` because it works for
+// "shared with me" folders in OneDrive Personal (See #2536, #2778)
+// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
+	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -378,9 +385,19 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var ok bool
-		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
-			dirCacheFoundRoot = true
+		var dirCacheRootIDExists bool
+		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		if f.root == "" {
+			// if f.root == "", it means f.root is the absolute root of the drive
+			// and its ID should have been found in NewFs
+			dirCacheFoundRoot = dirCacheRootIDExists
+		} else if _, err := f.dirCache.RootParentID(); err == nil {
+			// if root is in a folder, it must have a parent folder, and
+			// if dirCache has found root in NewFs, the parent folder's ID
+			// should be present.
+			// This RootParentID() check is a fix for #3164 which describes
+			// a possible case where the root is not found.
+			dirCacheFoundRoot = dirCacheRootIDExists
 		}
 	}
 
@@ -475,7 +492,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		driveID:   opt.DriveID,
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
-		pacer:     pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -703,9 +720,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			id := info.GetID()
 			f.dirCache.Put(remote, id)
 			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-			if folder != nil {
-				d.SetItems(folder.ChildCount)
-			}
+			d.SetItems(folder.ChildCount)
 			entries = append(entries, d)
 		} else {
 			o, err := f.newObjectWithInfo(remote, info)
@@ -819,9 +834,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -1340,12 +1352,12 @@ func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
 		}
 	} else {
 		opts = rest.Opts{
 			Method: "PATCH",
-			Path:   "/root:/" + rest.URLPathEscape(o.srvPath()),
+			Path:   "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
 		}
 	}
 	update := api.SetFileSystemInfo{
@@ -1491,23 +1503,40 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
 		return nil, errors.New("unknown-sized upload not supported")
 	}
 
+	uploadURLChan := make(chan string, 1)
+	gracefulCancel := func() {
+		uploadURL, ok := <-uploadURLChan
+		// Reading from uploadURLChan blocks the atexit process until
+		// we are able to use uploadURL to cancel the upload
+		if !ok { // createUploadSession failed - no need to cancel upload
+			return
+		}
+
+		fs.Debugf(o, "Cancelling multipart upload")
+		cancelErr := o.cancelUploadSession(uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+		}
+	}
+	cancelFuncHandle := atexit.Register(gracefulCancel)
+
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(modTime)
 	if err != nil {
+		close(uploadURLChan)
+		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
+	uploadURLChan <- uploadURL
 
-	// Cancel the session if something went wrong
 	defer func() {
 		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.cancelUploadSession(uploadURL)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+			fs.Debugf(o, "Error encountered during upload: %v", err)
+			gracefulCancel()
 		}
+		atexit.Unregister(cancelFuncHandle)
 	}()
 
 	// Upload the chunks
@@ -1668,6 +1697,21 @@ func getRelativePathInsideBase(base, target string) (string, bool) {
 	return "", false
 }
 
+// Adds a ":" at the end of `remotePath` in a proper manner.
+// If `remotePath` already ends with "/", change it to ":/"
+// If `remotePath` is "", return "".
+// A workaround for #2720 and #3039
+func withTrailingColon(remotePath string) string {
+	if remotePath == "" {
+		return ""
+	}
+
+	if strings.HasSuffix(remotePath, "/") {
+		return remotePath[:len(remotePath)-1] + ":/"
+	}
+	return remotePath + ":"
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -65,7 +65,7 @@ type Fs struct {
 	opt      Options            // parsed options
 	features *fs.Features       // optional features
 	srv      *rest.Client       // the connection to the server
-	pacer    *pacer.Pacer       // To pace and retry the API calls
+	pacer    *fs.Pacer          // To pace and retry the API calls
 	session  UserSessionInfo    // contains the session data
 	dirCache *dircache.DirCache // Map of directory path to directory id
 }
@@ -144,7 +144,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 
 	f.dirCache = dircache.New(root, "0", f)
@@ -287,9 +287,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 

backend/pcloud/pcloud.go

@@ -95,7 +95,7 @@ type Fs struct {
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // the connection to the server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *pacer.Pacer       // pacer for API calls
+	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
@@ -254,7 +254,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		root:  root,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         false,

backend/s3/s3.go

@@ -369,6 +369,10 @@ func init() {
 				Value:    "s3.us-west-1.wasabisys.com",
 				Help:     "Wasabi US West endpoint",
 				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-central-1.wasabisys.com",
+				Help:     "Wasabi EU Central endpoint",
+				Provider: "Wasabi",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -645,6 +649,9 @@ isn't set then "acl" is used instead.`,
 			}, {
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
+			}, {
+				Value: "DEEP_ARCHIVE",
+				Help:  "Glacier Deep Archive storage class",
 			}},
 		}, {
 			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
@@ -729,6 +736,14 @@ If it is set then rclone will use v2 authentication.
 Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "use_accelerate_endpoint",
+			Provider: "AWS",
+			Help: `If true use the AWS S3 accelerated endpoint.
+
+See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
+			Default:  false,
+			Advanced: true,
 		}},
 	})
 }
@@ -749,25 +764,26 @@ const (
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider             string        `config:"provider"`
-	EnvAuth              bool          `config:"env_auth"`
-	AccessKeyID          string        `config:"access_key_id"`
-	SecretAccessKey      string        `config:"secret_access_key"`
-	Region               string        `config:"region"`
-	Endpoint             string        `config:"endpoint"`
-	LocationConstraint   string        `config:"location_constraint"`
-	ACL                  string        `config:"acl"`
-	BucketACL            string        `config:"bucket_acl"`
-	ServerSideEncryption string        `config:"server_side_encryption"`
-	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
-	StorageClass         string        `config:"storage_class"`
-	UploadCutoff         fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum      bool          `config:"disable_checksum"`
-	SessionToken         string        `config:"session_token"`
-	UploadConcurrency    int           `config:"upload_concurrency"`
-	ForcePathStyle       bool          `config:"force_path_style"`
-	V2Auth               bool          `config:"v2_auth"`
+	Provider              string        `config:"provider"`
+	EnvAuth               bool          `config:"env_auth"`
+	AccessKeyID           string        `config:"access_key_id"`
+	SecretAccessKey       string        `config:"secret_access_key"`
+	Region                string        `config:"region"`
+	Endpoint              string        `config:"endpoint"`
+	LocationConstraint    string        `config:"location_constraint"`
+	ACL                   string        `config:"acl"`
+	BucketACL             string        `config:"bucket_acl"`
+	ServerSideEncryption  string        `config:"server_side_encryption"`
+	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
+	StorageClass          string        `config:"storage_class"`
+	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
+	DisableChecksum       bool          `config:"disable_checksum"`
+	SessionToken          string        `config:"session_token"`
+	UploadConcurrency     int           `config:"upload_concurrency"`
+	ForcePathStyle        bool          `config:"force_path_style"`
+	V2Auth                bool          `config:"v2_auth"`
+	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
 }
 
 // Fs represents a remote s3 server
@@ -782,7 +798,7 @@ type Fs struct {
 	bucketOKMu    sync.Mutex       // mutex to protect bucket OK
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
-	pacer         *pacer.Pacer     // To pace the API calls
+	pacer         *fs.Pacer        // To pace the API calls
 	srv           *http.Client     // a plain http client
 }
 
@@ -941,14 +957,15 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
-		WithS3ForcePathStyle(opt.ForcePathStyle)
+		WithS3ForcePathStyle(opt.ForcePathStyle).
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
@@ -1055,7 +1072,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		c:      c,
 		bucket: bucket,
 		ses:    ses,
-		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
+		pacer:  fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
@@ -1719,6 +1736,9 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	if o.fs.opt.SSEKMSKeyID != "" {
 		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
 	}
+	if o.fs.opt.StorageClass == "GLACIER" || o.fs.opt.StorageClass == "DEEP_ARCHIVE" {
+		return fs.ErrorCantSetModTime
+	}
 	if o.fs.opt.StorageClass != "" {
 		req.StorageClass = &o.fs.opt.StorageClass
 	}

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -14,6 +14,7 @@ import (
 	"os/user"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -25,6 +26,7 @@ import (
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/env"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
@@ -186,10 +188,10 @@ func readCurrentUser() (userName string) {
 	return os.Getenv("LOGNAME")
 }
 
-// Dial starts a client connection to the given SSH server. It is a
+// dial starts a client connection to the given SSH server. It is a
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
-func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
+func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
 	dialer := fshttp.NewDialer(fs.Config)
 	conn, err := dialer.Dial(network, addr)
 	if err != nil {
@@ -199,6 +201,7 @@ func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error
 	if err != nil {
 		return nil, err
 	}
+	fs.Debugf(f, "New connection %s->%s to %q", c.LocalAddr(), c.RemoteAddr(), c.ServerVersion())
 	return ssh.NewClient(c, chans, reqs), nil
 }
 
@@ -244,7 +247,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	c = &conn{
 		err: make(chan error, 1),
 	}
-	c.sshClient, err = Dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
+	c.sshClient, err = f.dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't connect SSH")
 	}
@@ -315,18 +318,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
-// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
-// variables afterwards.
-func shellExpand(s string) string {
-	if s != "" {
-		if s[0] == '~' {
-			s = "${HOME}" + s[1:]
-		}
-		s = os.ExpandEnv(s)
-	}
-	return s
-}
-
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -347,6 +338,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		Auth:            []ssh.AuthMethod{},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 		Timeout:         fs.Config.ConnectTimeout,
+		ClientVersion:   "SSH-2.0-" + fs.Config.UserAgent,
 	}
 
 	if opt.UseInsecureCipher {
@@ -354,7 +346,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
-	keyFile := shellExpand(opt.KeyFile)
+	keyFile := env.ShellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
 	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
@@ -427,6 +419,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
+	return NewFsWithConnection(name, root, opt, sshConfig)
+}
+
+// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
+// the host specified in the ssh.ClientConfig
+func NewFsWithConnection(name string, root string, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
 		name:      name,
 		root:      root,
@@ -807,6 +805,49 @@ func (f *Fs) Hashes() hash.Set {
 	return set
 }
 
+// About gets usage stats
+func (f *Fs) About() (*fs.Usage, error) {
+	c, err := f.getSftpConnection()
+	if err != nil {
+		return nil, errors.Wrap(err, "About get SFTP connection")
+	}
+	session, err := c.sshClient.NewSession()
+	f.putSftpConnection(&c, err)
+	if err != nil {
+		return nil, errors.Wrap(err, "About put SFTP connection")
+	}
+
+	var stdout, stderr bytes.Buffer
+	session.Stdout = &stdout
+	session.Stderr = &stderr
+	escapedPath := shellEscape(f.root)
+	if f.opt.PathOverride != "" {
+		escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
+	}
+	if len(escapedPath) == 0 {
+		escapedPath = "/"
+	}
+	err = session.Run("df -k " + escapedPath)
+	if err != nil {
+		_ = session.Close()
+		return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.")
+	}
+	_ = session.Close()
+
+	usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes())
+	usage := &fs.Usage{}
+	if usageTotal >= 0 {
+		usage.Total = fs.NewUsageValue(usageTotal)
+	}
+	if usageUsed >= 0 {
+		usage.Used = fs.NewUsageValue(usageUsed)
+	}
+	if usageAvail >= 0 {
+		usage.Free = fs.NewUsageValue(usageAvail)
+	}
+	return usage, nil
+}
+
 // Fs is the filesystem this remote sftp file object is located within
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -897,6 +938,35 @@ func parseHash(bytes []byte) string {
 	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
 }
 
+// Parses the byte array output from the SSH session
+// returned by an invocation of df into
+// the disk size, used space, and avaliable space on the disk, in that order.
+// Only works when `df` has output info on only one disk
+func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
+	spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
+	lines := strings.Split(string(bytes), "\n")
+	if len(lines) < 2 {
+		return
+	}
+	split := strings.Fields(lines[1])
+	if len(split) < 6 {
+		return
+	}
+	spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
+	if err != nil {
+		spaceTotal = -1
+	}
+	spaceUsed, err = strconv.ParseInt(split[2], 10, 64)
+	if err != nil {
+		spaceUsed = -1
+	}
+	spaceAvail, err = strconv.ParseInt(split[3], 10, 64)
+	if err != nil {
+		spaceAvail = -1
+	}
+	return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
+}
+
 // Size returns the size in bytes of the remote sftp file
 func (o *Object) Size() int64 {
 	return o.size

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -35,3 +35,17 @@ func TestParseHash(t *testing.T) {
 		assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
 	}
 }
+
+func TestParseUsage(t *testing.T) {
+	for i, test := range []struct {
+		sshOutput string
+		usage     [3]int64
+	}{
+		{"Filesystem     1K-blocks     Used Available Use% Mounted on\n/dev/root       91283092 81111888  10154820  89% /", [3]int64{93473886208, 83058573312, 10398535680}},
+		{"Filesystem     1K-blocks  Used Available Use% Mounted on\ntmpfs             818256  1636    816620   1% /run", [3]int64{837894144, 1675264, 836218880}},
+		{"Filesystem   1024-blocks     Used Available Capacity iused      ifree %iused  Mounted on\n/dev/disk0s2   244277768 94454848 149566920    39%  997820 4293969459    0%   /", [3]int64{250140434432, 96721764352, 153156526080}},
+	} {
+		gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput))
+		assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
+	}
+}

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.9
+// +build plan9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/sftp/stringlock_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/swift/auth.go

@@ -2,6 +2,7 @@ package swift
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/ncw/swift"
 )
@@ -65,6 +66,14 @@ func (a *auth) Token() string {
 	return a.parentAuth.Token()
 }
 
+// Expires returns the time the token expires if known or Zero if not.
+func (a *auth) Expires() (t time.Time) {
+	if do, ok := a.parentAuth.(swift.Expireser); ok {
+		t = do.Expires()
+	}
+	return t
+}
+
 // The CDN url if available
 func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
@@ -74,4 +83,7 @@ func (a *auth) CdnUrl() string { // nolint
 }
 
 // Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)
+var (
+	_ swift.Authenticator = (*auth)(nil)
+	_ swift.Expireser     = (*auth)(nil)
+)

backend/swift/swift.go

@@ -216,7 +216,7 @@ type Fs struct {
 	containerOK       bool              // true if we have created the container
 	segmentsContainer string            // container to store the segments (if any) in
 	noCheckContainer  bool              // don't check the container before creating it
-	pacer             *pacer.Pacer      // To pace the API calls
+	pacer             *fs.Pacer         // To pace the API calls
 }
 
 // Object describes a swift object
@@ -286,6 +286,31 @@ func shouldRetry(err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
+// shouldRetryHeaders returns a boolean as to whether this err
+// deserves to be retried.  It reads the headers passed in looking for
+// `Retry-After`. It returns the err as a convenience
+func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
+	if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
+		if value := headers["Retry-After"]; value != "" {
+			retryAfter, parseErr := strconv.Atoi(value)
+			if parseErr != nil {
+				fs.Errorf(nil, "Failed to parse Retry-After: %q: %v", value, parseErr)
+			} else {
+				duration := time.Second * time.Duration(retryAfter)
+				if duration <= 60*time.Second {
+					// Do a short sleep immediately
+					fs.Debugf(nil, "Sleeping for %v to obey Retry-After", duration)
+					time.Sleep(duration)
+					return true, err
+				}
+				// Delay a long sleep for a retry
+				return false, fserrors.NewErrorRetryAfter(duration)
+			}
+		}
+	}
+	return shouldRetry(err)
+}
+
 // Pattern to match a swift path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -401,7 +426,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		segmentsContainer: container + "_segments",
 		root:              directory,
 		noCheckContainer:  noCheckContainer,
-		pacer:             pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
+		pacer:             fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -413,8 +438,9 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		// Check to see if the object exists - ignoring directory markers
 		var info swift.Object
 		err = f.pacer.Call(func() (bool, error) {
-			info, _, err = f.c.Object(container, directory)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			info, rxHeaders, err = f.c.Object(container, directory)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			f.root = path.Dir(directory)
@@ -723,8 +749,9 @@ func (f *Fs) Mkdir(dir string) error {
 	var err error = swift.ContainerNotFound
 	if !f.noCheckContainer {
 		err = f.pacer.Call(func() (bool, error) {
-			_, _, err = f.c.Container(f.container)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			_, rxHeaders, err = f.c.Container(f.container)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 	}
 	if err == swift.ContainerNotFound {
@@ -816,8 +843,9 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	srcFs := srcObj.fs
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err != nil {
 		return nil, err
@@ -927,7 +955,7 @@ func (o *Object) readMetaData() (err error) {
 	var h swift.Headers
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, h, err = o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
-		return shouldRetry(err)
+		return shouldRetryHeaders(h, err)
 	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
@@ -1002,8 +1030,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	headers := fs.OpenOptionHeaders(options)
 	_, isRanging := headers["Range"]
 	err = o.fs.pacer.Call(func() (bool, error) {
-		in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		in, rxHeaders, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return
 }
@@ -1074,8 +1103,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	// Create the segmentsContainer if it doesn't exist
 	var err error
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		_, rxHeaders, err = o.fs.c.Container(o.fs.segmentsContainer)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
@@ -1115,8 +1145,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 		segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
 		fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			rxHeaders, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return "", err
@@ -1129,8 +1160,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	emptyReader := bytes.NewReader(nil)
 	manifestName := o.fs.root + o.remote
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return uniquePrefix + "/", err
 }
@@ -1174,7 +1206,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		var rxHeaders swift.Headers
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
-			return shouldRetry(err)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return err

backend/swift/swift_internal_test.go

@@ -1,6 +1,13 @@
 package swift
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/swift"
+	"github.com/stretchr/testify/assert"
+)
 
 func TestInternalUrlEncode(t *testing.T) {
 	for _, test := range []struct {
@@ -23,3 +30,37 @@ func TestInternalUrlEncode(t *testing.T) {
 		}
 	}
 }
+
+func TestInternalShouldRetryHeaders(t *testing.T) {
+	headers := swift.Headers{
+		"Content-Length": "64",
+		"Content-Type":   "text/html; charset=UTF-8",
+		"Date":           "Mon: 18 Mar 2019 12:11:23 GMT",
+		"Retry-After":    "1",
+	}
+	err := &swift.Error{
+		StatusCode: 429,
+		Text:       "Too Many Requests",
+	}
+
+	// Short sleep should just do the sleep
+	start := time.Now()
+	retry, gotErr := shouldRetryHeaders(headers, err)
+	dt := time.Since(start)
+	assert.True(t, retry)
+	assert.Equal(t, err, gotErr)
+	assert.True(t, dt > time.Second/2)
+
+	// Long sleep should return RetryError
+	headers["Retry-After"] = "3600"
+	start = time.Now()
+	retry, gotErr = shouldRetryHeaders(headers, err)
+	dt = time.Since(start)
+	assert.True(t, dt < time.Second)
+	assert.False(t, retry)
+	assert.Equal(t, true, fserrors.IsRetryAfterError(gotErr))
+	after := gotErr.(fserrors.RetryAfter).RetryAfter()
+	dt = after.Sub(start)
+	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
+
+}

backend/union/union.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/hash"
@@ -342,7 +343,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if configName != "local" {
 			rootString = configName + ":" + rootString
 		}
-		myFs, err := fs.NewFs(rootString)
+		myFs, err := cache.Get(rootString)
 		if err != nil {
 			if err == fs.ErrorIsFile {
 				return myFs, err

backend/webdav/webdav.go

@@ -101,7 +101,7 @@ type Fs struct {
 	endpoint           *url.URL      // URL of the host
 	endpointURL        string        // endpoint as a string
 	srv                *rest.Client  // the connection to the one drive server
-	pacer              *pacer.Pacer  // pacer for API calls
+	pacer              *fs.Pacer     // pacer for API calls
 	precision          time.Duration // mod time precision
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
@@ -146,6 +146,7 @@ func (f *Fs) Features() *fs.Features {
 
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
+	423, // Locked
 	429, // Too Many Requests.
 	500, // Internal Server Error
 	502, // Bad Gateway
@@ -318,7 +319,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		endpoint:    u,
 		endpointURL: u.String(),
 		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		precision:   fs.ModTimeNotSupported,
 	}
 	f.features = (&fs.Features{
@@ -644,10 +645,18 @@ func (f *Fs) _mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	return f.pacer.Call(func() (bool, error) {
+	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// already exists
+		// owncloud returns 423/StatusLocked if the create is already in progress
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
+			return nil
+		}
+	}
+	return err
 }
 
 // mkdir makes the directory and parents using native paths
@@ -655,12 +664,7 @@ func (f *Fs) mkdir(dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
-		// already exists
-		// owncloud returns 423/StatusLocked if the create is already in progress
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
-			return nil
-		}
-		// parent does not exist
+		// parent does not exist so create it first then try again
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
@@ -913,11 +917,13 @@ func (f *Fs) About() (*fs.Usage, error) {
 		return nil, errors.Wrap(err, "about call failed")
 	}
 	usage := &fs.Usage{}
-	if q.Available >= 0 && q.Used >= 0 {
-		usage.Total = fs.NewUsageValue(q.Available + q.Used)
-	}
-	if q.Used >= 0 {
-		usage.Used = fs.NewUsageValue(q.Used)
+	if q.Available != 0 || q.Used != 0 {
+		if q.Available >= 0 && q.Used >= 0 {
+			usage.Total = fs.NewUsageValue(q.Available + q.Used)
+		}
+		if q.Used >= 0 {
+			usage.Used = fs.NewUsageValue(q.Used)
+		}
 	}
 	return usage, nil
 }

backend/yandex/yandex.go

@@ -93,7 +93,7 @@ type Fs struct {
 	opt      Options      // parsed options
 	features *fs.Features // optional features
 	srv      *rest.Client // the connection to the yandex server
-	pacer    *pacer.Pacer // pacer for API calls
+	pacer    *fs.Pacer    // pacer for API calls
 	diskRoot string       // root path with "disk:/" container name
 }
 
@@ -269,7 +269,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		opt:   *opt,
 		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{

bin/get-github-release.go

@@ -17,14 +17,18 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/lib/rest"
+	"golang.org/x/net/html"
 	"golang.org/x/sys/unix"
 )
 
@@ -33,6 +37,7 @@ var (
 	install = flag.Bool("install", false, "Install the downloaded package using sudo dpkg -i.")
 	extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.")
 	bindir  = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.")
+	useAPI  = flag.Bool("use-api", false, "Use the API for finding the release instead of scraping the page.")
 	// Globals
 	matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
 	osAliases    = map[string][]string{
@@ -209,6 +214,57 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	return "", ""
 }
 
+// Get an asset URL and name by scraping the downloads page
+//
+// This doesn't use the API so isn't rate limited when not using GITHUB login details
+func getAssetFromReleasesPage(project string, matchName *regexp.Regexp) (assetURL string, assetName string) {
+	baseURL := "https://github.com/" + project + "/releases"
+	log.Printf("Fetching asset info for %q from %q", project, baseURL)
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		log.Fatalf("URL Parse failed: %v", err)
+	}
+	resp, err := http.Get(baseURL)
+	if err != nil {
+		log.Fatalf("Failed to fetch release info %q: %v", baseURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("Error: %s", readBody(resp.Body))
+		log.Fatalf("Bad status %d when fetching %q release info: %s", resp.StatusCode, baseURL, resp.Status)
+	}
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		log.Fatalf("Failed to parse web page: %v", err)
+	}
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			for _, a := range n.Attr {
+				if a.Key == "href" {
+					if name := path.Base(a.Val); matchName.MatchString(name) && isOurOsArch(name) {
+						if u, err := rest.URLJoin(base, a.Val); err == nil {
+							if assetName == "" {
+								assetName = name
+								assetURL = u.String()
+							}
+						}
+					}
+					break
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	if assetName == "" || assetURL == "" {
+		log.Fatalf("Didn't find URL in page")
+	}
+	return assetURL, assetName
+}
+
 // isOurOsArch returns true if s contains our OS and our Arch
 func isOurOsArch(s string) bool {
 	s = strings.ToLower(s)
@@ -346,7 +402,12 @@ func main() {
 		log.Fatalf("Invalid regexp for name %q: %v", nameRe, err)
 	}
 
-	assetURL, assetName := getAsset(project, matchName)
+	var assetURL, assetName string
+	if *useAPI {
+		assetURL, assetName = getAsset(project, matchName)
+	} else {
+		assetURL, assetName = getAssetFromReleasesPage(project, matchName)
+	}
 	fileName := filepath.Join(os.TempDir(), assetName)
 	getFile(assetURL, fileName)
 

bin/make_changelog.py

@@ -12,8 +12,8 @@ from collections import defaultdict
 
 IGNORE_RES = [
     r"^Add .* to contributors$",
-    r"^Start v\d+.\d+-DEV development$",
-    r"^Version v\d.\d+$",
+    r"^Start v\d+\.\d+(\.\d+)?-DEV development$",
+    r"^Version v\d+\.\d+(\.\d+)?$",
 ]
 
 IGNORE_RE = re.compile("(?:" + "|".join(IGNORE_RES) + ")")

bin/make_manual.py

@@ -36,6 +36,7 @@ docs = [
     "http.md",
     "hubic.md",
     "jottacloud.md",
+    "koofr.md",
     "mega.md",
     "azureblob.md",
     "onedrive.md",

cmd/about/about.go

@@ -95,6 +95,9 @@ Use the --json flag for a computer readable output, eg
 			if err != nil {
 				return errors.Wrap(err, "About call failed")
 			}
+			if u == nil {
+				return errors.New("nil usage returned")
+			}
 			if jsonOutput {
 				out := json.NewEncoder(os.Stdout)
 				out.SetIndent("", "\t")

cmd/cmd.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configflags"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/filter"
@@ -83,7 +84,7 @@ func NewFsFile(remote string) (fs.Fs, string) {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
-	f, err := fs.NewFs(remote)
+	f, err := cache.Get(remote)
 	switch err {
 	case fs.ErrorIsFile:
 		return f, path.Base(fsPath)
@@ -131,7 +132,7 @@ func NewFsSrc(args []string) fs.Fs {
 //
 // This must point to a directory
 func newFsDir(remote string) fs.Fs {
-	f, err := fs.NewFs(remote)
+	f, err := cache.Get(remote)
 	if err != nil {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
@@ -180,7 +181,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 			log.Fatalf("%q is a directory", args[1])
 		}
 	}
-	fdst, err := fs.NewFs(dstRemote)
+	fdst, err := cache.Get(dstRemote)
 	switch err {
 	case fs.ErrorIsFile:
 		fs.CountError(err)
@@ -230,22 +231,34 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	SigInfoHandler()
 	for try := 1; try <= *retries; try++ {
 		err = f()
-		if !Retry || (err == nil && !accounting.Stats.Errored()) {
+		fs.CountError(err)
+		lastErr := accounting.Stats.GetLastError()
+		if err == nil {
+			err = lastErr
+		}
+		if !Retry || !accounting.Stats.Errored() {
 			if try > 1 {
 				fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries)
 			}
 			break
 		}
-		if fserrors.IsFatalError(err) || accounting.Stats.HadFatalError() {
+		if accounting.Stats.HadFatalError() {
 			fs.Errorf(nil, "Fatal error received - not attempting retries")
 			break
 		}
-		if fserrors.IsNoRetryError(err) || (accounting.Stats.Errored() && !accounting.Stats.HadRetryError()) {
+		if accounting.Stats.Errored() && !accounting.Stats.HadRetryError() {
 			fs.Errorf(nil, "Can't retry this error - not attempting retries")
 			break
 		}
-		if err != nil {
-			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), err)
+		if retryAfter := accounting.Stats.RetryAfter(); !retryAfter.IsZero() {
+			d := retryAfter.Sub(time.Now())
+			if d > 0 {
+				fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
+				time.Sleep(d)
+			}
+		}
+		if lastErr != nil {
+			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), lastErr)
 		} else {
 			fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.Stats.GetErrors())
 		}
@@ -258,7 +271,12 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	}
 	stopStats()
 	if err != nil {
-		log.Printf("Failed to %s: %v", cmd.Name(), err)
+		nerrs := accounting.Stats.GetErrors()
+		if nerrs <= 1 {
+			log.Printf("Failed to %s: %v", cmd.Name(), err)
+		} else {
+			log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, err)
+		}
 		resolveExitCode(err)
 	}
 	if showStats && (accounting.Stats.Errored() || *statsInterval > 0) {
@@ -341,8 +359,7 @@ func initConfig() {
 	configflags.SetFlags()
 
 	// Load filters
-	var err error
-	filter.Active, err = filter.NewFilter(&filterflags.Opt)
+	err := filterflags.Reload()
 	if err != nil {
 		log.Fatalf("Failed to load filters: %v", err)
 	}

cmd/cmount/mount.go

@@ -127,7 +127,7 @@ func waitFor(fn func() bool) (ok bool) {
 func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
-	// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
+	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
 	if runtime.GOOS != "windows" {
 		fi, err := os.Stat(mountpoint)
 		if err != nil {

cmd/config/config.go

@@ -98,6 +98,9 @@ Note that if the config process would normally ask a question the
 default is taken.  Each time that happens rclone will print a message
 saying how to affect the value taken.
 
+If any of the parameters passed is a password field, then rclone will
+automatically obscure them before putting them in the config file.
+
 So for example if you wanted to configure a Google Drive remote but
 using remote authorization you would do this:
 
@@ -125,10 +128,14 @@ var configUpdateCommand = &cobra.Command{
 Update an existing remote's options. The options should be passed in
 in pairs of <key> <value>.
 
-For example to update the env_auth field of a remote of name myremote you would do:
+For example to update the env_auth field of a remote of name myremote
+you would do:
 
     rclone config update myremote swift env_auth true
 
+If any of the parameters passed is a password field, then rclone will
+automatically obscure them before putting them in the config file.
+
 If the remote uses oauth the token will be updated, if you don't
 require this add an extra parameter thus:
 
@@ -168,6 +175,9 @@ should be passed in in pairs of <key> <value>.
 For example to set password of a remote of name myremote you would do:
 
     rclone config password myremote fieldname mypassword
+
+This command is obsolete now that "config update" and "config create"
+both support obscuring passwords directly.
 `,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(3, 256, command, args)

cmd/copy/copy.go

@@ -7,8 +7,13 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	createEmptySrcDirs = false
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy")
 }
 
 var commandDefintion = &cobra.Command{
@@ -69,7 +74,7 @@ changed recently very efficiently like this:
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc)
+				return sync.CopyDir(fdst, fsrc, createEmptySrcDirs)
 			}
 			return operations.CopyFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/copyto/copyto.go

@@ -48,7 +48,7 @@ destination.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.CopyDir(fdst, fsrc)
+				return sync.CopyDir(fdst, fsrc, false)
 			}
 			return operations.CopyFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/gendocs/gendocs.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ncw/rclone/cmd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
+	"github.com/spf13/pflag"
 )
 
 func init() {
@@ -50,6 +51,10 @@ rclone.org website.`,
 			base := strings.TrimSuffix(name, path.Ext(name))
 			return "/commands/" + strings.ToLower(base) + "/"
 		}
+		// Hide all of the root entries flags
+		cmd.Root.Flags().VisitAll(func(flag *pflag.Flag) {
+			flag.Hidden = true
+		})
 		return doc.GenMarkdownTreeCustom(cmd.Root, out, prepender, linkHandler)
 	},
 }

cmd/help.go

@@ -45,7 +45,7 @@ __rclone_custom_func() {
         else
             __rclone_init_completion -n : || return
         fi
-        if [[ $cur =~ ^[[:alnum:]]*$ ]]; then
+        if [[ $cur != *:* ]]; then
             local remote
             while IFS= read -r remote; do
                 [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
@@ -54,10 +54,10 @@ __rclone_custom_func() {
                 local paths=("$cur"*)
                 [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
             fi
-        elif [[ $cur =~ ^[[:alnum:]]+: ]]; then
+        else
             local path=${cur#*:}
             if [[ $path == */* ]]; then
-                local prefix=${path%/*}
+                local prefix=$(eval printf '%s' "${path%/*}")
             else
                 local prefix=
             fi
@@ -66,6 +66,7 @@ __rclone_custom_func() {
                 local reply=${prefix:+$prefix/}$line
                 [[ $reply != $path* ]] || COMPREPLY+=("$reply")
             done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
+	    [[ ! ${COMPREPLY[@]} ]] || compopt -o filenames
         fi
         [[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
     fi

cmd/ls/lshelp/lshelp.go

@@ -2,7 +2,7 @@ package lshelp
 
 // Help describes the common help for all the list commands
 var Help = `
-Any of the filtering options can be applied to this commmand.
+Any of the filtering options can be applied to this command.
 
 There are several related list commands
 

cmd/lsf/lsf.go

@@ -10,7 +10,6 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/operations"
-	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -67,8 +66,11 @@ output:
     s - size
     t - modification time
     h - hash
-    i - ID of object if known
+    i - ID of object
+    o - Original ID of underlying object
     m - MimeType of object if known
+    e - encrypted name
+    T - tier of storage if known, eg "Hot" or "Cool"
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -161,6 +163,12 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 	list.SetCSV(csv)
 	list.SetDirSlash(dirSlash)
 	list.SetAbsolute(absolute)
+	var opt = operations.ListJSONOpt{
+		NoModTime: true,
+		DirsOnly:  dirsOnly,
+		FilesOnly: filesOnly,
+		Recurse:   recurse,
+	}
 
 	for _, char := range format {
 		switch char {
@@ -168,38 +176,31 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 			list.AddPath()
 		case 't':
 			list.AddModTime()
+			opt.NoModTime = false
 		case 's':
 			list.AddSize()
 		case 'h':
 			list.AddHash(hashType)
+			opt.ShowHash = true
 		case 'i':
 			list.AddID()
 		case 'm':
 			list.AddMimeType()
+		case 'e':
+			list.AddEncrypted()
+			opt.ShowEncrypted = true
+		case 'o':
+			list.AddOrigID()
+			opt.ShowOrigIDs = true
+		case 'T':
+			list.AddTier()
 		default:
 			return errors.Errorf("Unknown format character %q", char)
 		}
 	}
 
-	return walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(path string, entries fs.DirEntries, err error) error {
-		if err != nil {
-			fs.CountError(err)
-			fs.Errorf(path, "error listing: %v", err)
-			return nil
-		}
-		for _, entry := range entries {
-			_, isDir := entry.(fs.Directory)
-			if isDir {
-				if filesOnly {
-					continue
-				}
-			} else {
-				if dirsOnly {
-					continue
-				}
-			}
-			_, _ = fmt.Fprintln(out, list.Format(entry))
-		}
+	return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
+		_, _ = fmt.Fprintln(out, list.Format(item))
 		return nil
 	})
 }

cmd/lsjson/lsjson.go

@@ -23,6 +23,8 @@ func init() {
 	commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
 	commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.")
 	commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
+	commandDefintion.Flags().BoolVarP(&opt.FilesOnly, "files-only", "", false, "Show only files in the listing.")
+	commandDefintion.Flags().BoolVarP(&opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing.")
 }
 
 var commandDefintion = &cobra.Command{
@@ -40,13 +42,16 @@ The output is an array of Items, where each Item looks like this
       },
       "ID": "y2djkhiujf83u33",
       "OrigID": "UYOJVTUW00Q1RzTDA",
+      "IsBucket" : false,
       "IsDir" : false,
       "MimeType" : "application/octet-stream",
       "ModTime" : "2017-05-31T16:15:57.034468261+01:00",
       "Name" : "file.txt",
       "Encrypted" : "v0qpsdq8anpci8n929v3uu9338",
+      "EncryptedPath" : "kja9098349023498/v0qpsdq8anpci8n929v3uu9338",
       "Path" : "full/path/goes/here/file.txt",
-      "Size" : 6
+      "Size" : 6,
+      "Tier" : "hot",
    }
 
 If --hash is not specified the Hashes property won't be emitted.
@@ -55,11 +60,19 @@ If --no-modtime is specified then ModTime will be blank.
 
 If --encrypted is not specified the Encrypted won't be emitted.
 
+If --dirs-only is not specified files in addition to directories are returned
+
+If --files-only is not specified directories in addition to the files will be returned.
+
 The Path field will only show folders below the remote path being listed.
 If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
 will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
 When used without --recursive the Path will always be the same as Name.
 
+If the directory is a bucket in a bucket based backend, then
+"IsBucket" will be set to true. This key won't be present unless it is
+"true".
+
 The time is in RFC3339 format with up to nanosecond precision.  The
 number of decimal digits in the seconds will depend on the precision
 that the remote can hold the times, so if times are accurate to the

cmd/mount/dir.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // Dir represents a directory entry
@@ -20,7 +20,7 @@ type Dir struct {
 	*vfs.Dir
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.Node = (*Dir)(nil)
 
 // Attr updates the attributes of a directory

cmd/mount/file.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"io"
 	"time"
 
@@ -11,7 +12,6 @@ import (
 	"github.com/ncw/rclone/cmd/mountlib"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // File represents a file

cmd/mount/fs.go

@@ -5,6 +5,7 @@
 package mount
 
 import (
+	"context"
 	"syscall"
 
 	"bazil.org/fuse"
@@ -15,7 +16,6 @@ import (
 	"github.com/ncw/rclone/vfs"
 	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FS represents the top level filing system
@@ -24,7 +24,7 @@ type FS struct {
 	f fs.Fs
 }
 
-// Check interface satistfied
+// Check interface satisfied
 var _ fusefs.FS = (*FS)(nil)
 
 // NewFS makes a new FS
@@ -46,7 +46,7 @@ func (f *FS) Root() (node fusefs.Node, err error) {
 	return &Dir{root}, nil
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.FSStatfser = (*FS)(nil)
 
 // Statfs is called to obtain file system metadata.

cmd/mount/handle.go

@@ -3,13 +3,13 @@
 package mount
 
 import (
+	"context"
 	"io"
 
 	"bazil.org/fuse"
 	fusefs "bazil.org/fuse/fs"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FileHandle is an open for read file handle on a File

cmd/move/move.go

@@ -10,11 +10,13 @@ import (
 // Globals
 var (
 	deleteEmptySrcDirs = false
+	createEmptySrcDirs = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
 	commandDefintion.Flags().BoolVarP(&deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move")
+	commandDefintion.Flags().BoolVarP(&createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move")
 }
 
 var commandDefintion = &cobra.Command{
@@ -52,7 +54,7 @@ can speed transfers up greatly.
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs)
+				return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
 			}
 			return operations.MoveFile(fdst, fsrc, srcFileName, srcFileName)
 		})

cmd/moveto/moveto.go

@@ -19,7 +19,7 @@ If source:path is a file or directory then it moves it to a file or
 directory named dest:path.
 
 This can be used to rename files or upload single files to other than
-their existing name.  If the source is a directory then it acts exacty
+their existing name.  If the source is a directory then it acts exactly
 like the move command.
 
 So
@@ -52,7 +52,7 @@ transfer.
 
 		cmd.Run(true, true, command, func() error {
 			if srcFileName == "" {
-				return sync.MoveDir(fdst, fsrc, false)
+				return sync.MoveDir(fdst, fsrc, false, false)
 			}
 			return operations.MoveFile(fdst, fsrc, dstFileName, srcFileName)
 		})

cmd/ncdu/ncdu.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strings"
 
+	runewidth "github.com/mattn/go-runewidth"
 	"github.com/ncw/rclone/cmd"
 	"github.com/ncw/rclone/cmd/ncdu/scan"
 	"github.com/ncw/rclone/fs"
@@ -122,7 +123,7 @@ func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface
 func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
 	for _, c := range msg {
 		termbox.SetCell(x, y, c, fg, bg)
-		x++
+		x += runewidth.RuneWidth(c)
 		if x >= xmax {
 			return
 		}
@@ -360,7 +361,7 @@ func (u *UI) Draw() error {
 		Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %v, Objects: %d%s", fs.SizeSuffix(size), count, message)
 	}
 
-	// Show the box on top if requred
+	// Show the box on top if required
 	if u.showBox {
 		u.Box()
 	}

cmd/progress.go

@@ -82,7 +82,7 @@ var (
 	progressMu sync.Mutex
 )
 
-// printProgress prings the progress with an optional log
+// printProgress prints the progress with an optional log
 func printProgress(logMessage string) {
 	progressMu.Lock()
 	defer progressMu.Unlock()

cmd/rc/rc.go

@@ -24,6 +24,7 @@ var (
 	jsonInput = ""
 	authUser  = ""
 	authPass  = ""
+	loopback  = false
 )
 
 func init() {
@@ -33,6 +34,7 @@ func init() {
 	commandDefintion.Flags().StringVarP(&jsonInput, "json", "", jsonInput, "Input JSON - use instead of key=value args.")
 	commandDefintion.Flags().StringVarP(&authUser, "user", "", "", "Username to use to rclone remote control.")
 	commandDefintion.Flags().StringVarP(&authPass, "pass", "", "", "Password to use to connect to rclone remote control.")
+	commandDefintion.Flags().BoolVarP(&loopback, "loopback", "", false, "If set connect to this rclone instance not via HTTP.")
 }
 
 var commandDefintion = &cobra.Command{
@@ -58,6 +60,12 @@ The --json parameter can be used to pass in a JSON blob as an input
 instead of key=value arguments.  This is the only way of passing in
 more complicated values.
 
+Use --loopback to connect to the rclone instance running "rclone rc".
+This is very useful for testing commands without having to run an
+rclone rc server, eg:
+
+    rclone rc --loopback operations/about fs=/
+
 Use "rclone rc" to see a list of all possible commands.`,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1E9, command, args)
@@ -102,6 +110,15 @@ func setAlternateFlag(flagName string, output *string) {
 //
 // if err is set, out may be a valid error return or it may be nil
 func doCall(path string, in rc.Params) (out rc.Params, err error) {
+	// If loopback set, short circuit HTTP request
+	if loopback {
+		call := rc.Calls.Get(path)
+		if call == nil {
+			return nil, errors.Errorf("method %q not found", path)
+		}
+		return call.Fn(in)
+	}
+
 	// Do HTTP request
 	client := fshttp.NewClient(fs.Config)
 	url += path

cmd/serve/dlna/cd-service-desc.go

@@ -1,451 +0,0 @@
-package dlna
-
-const contentDirectoryServiceDescription = `<?xml version="1.0"?>
-<scpd xmlns="urn:schemas-upnp-org:service-1-0">
-  <specVersion>
-    <major>1</major>
-    <minor>0</minor>
-  </specVersion>
-  <actionList>
-    <action>
-      <name>GetSearchCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SearchCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SearchCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortExtensionCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortExtensionCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetFeatureList</name>
-      <argumentList>
-        <argument>
-          <name>FeatureList</name>
-          <direction>out</direction>
-          <relatedStateVariable>FeatureList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSystemUpdateID</name>
-      <argumentList>
-        <argument>
-          <name>Id</name>
-          <direction>out</direction>
-          <relatedStateVariable>SystemUpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Browse</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>BrowseFlag</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Search</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SearchCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateObject</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Elements</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DestroyObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>UpdateObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>CurrentTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>MoveObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewParentID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ImportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ExportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>StopTransferResource</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DeleteResource</name>
-      <argumentList>
-        <argument>
-          <name>ResourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetTransferProgress</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferStatus</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferLength</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferTotal</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateReference</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-  </actionList>
-  <serviceStateTable>
-    <stateVariable sendEvents="no">
-      <name>SearchCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortExtensionCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>SystemUpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>ContainerUpdateIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>TransferIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>FeatureList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ObjectID</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Result</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SearchCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_BrowseFlag</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>BrowseMetadata</allowedValue>
-        <allowedValue>BrowseDirectChildren</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Filter</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SortCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Index</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Count</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_UpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferStatus</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>COMPLETED</allowedValue>
-        <allowedValue>ERROR</allowedValue>
-        <allowedValue>IN_PROGRESS</allowedValue>
-        <allowedValue>STOPPED</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferLength</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferTotal</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TagValueList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_URI</name>
-      <dataType>uri</dataType>
-    </stateVariable>
-  </serviceStateTable>
-</scpd>`

cmd/serve/dlna/cds.go

@@ -9,11 +9,13 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"sort"
 
 	"github.com/anacrolix/dms/dlna"
 	"github.com/anacrolix/dms/upnp"
 	"github.com/anacrolix/dms/upnpav"
+	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/vfs"
 	"github.com/pkg/errors"
 )
@@ -27,6 +29,8 @@ func (cds *contentDirectoryService) updateIDString() string {
 	return fmt.Sprintf("%d", uint32(os.Getpid()))
 }
 
+var mediaMimeTypeRegexp = regexp.MustCompile("^(video|audio|image)/")
+
 // Turns the given entry and DMS host into a UPnP object. A nil object is
 // returned if the entry is not of interest.
 func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fileInfo os.FileInfo, host string) (ret interface{}, err error) {
@@ -47,8 +51,13 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
 		return
 	}
 
-	// Hardcode "videoItem" so that files show up in VLC.
-	obj.Class = "object.item.videoItem"
+	mimeType := fs.MimeTypeFromName(fileInfo.Name())
+	mediaType := mediaMimeTypeRegexp.FindStringSubmatch(mimeType)
+	if mediaType == nil {
+		return
+	}
+
+	obj.Class = "object.item." + mediaType[1] + "Item"
 	obj.Title = fileInfo.Name()
 
 	item := upnpav.Item{
@@ -65,8 +74,7 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
 				"path": {cdsObject.Path},
 			}.Encode(),
 		}).String(),
-		// Hardcode "video/x-matroska" so that files show up in VLC.
-		ProtocolInfo: fmt.Sprintf("http-get:*:video/x-matroska:%s", dlna.ContentFeatures{
+		ProtocolInfo: fmt.Sprintf("http-get:*:%s:%s", mimeType, dlna.ContentFeatures{
 			SupportRange: true,
 		}.String()),
 		Bitrate:    0,
@@ -106,14 +114,14 @@ func (cds *contentDirectoryService) readContainer(o object, host string) (ret []
 		}
 		obj, err := cds.cdsObjectToUpnpavObject(child, de, host)
 		if err != nil {
-			log.Printf("error with %s: %s", child.FilePath(), err)
+			fs.Errorf(cds, "error with %s: %s", child.FilePath(), err)
 			continue
 		}
-		if obj != nil {
-			ret = append(ret, obj)
-		} else {
-			log.Printf("bad %s", de)
+		if obj == nil {
+			fs.Debugf(cds, "unrecognized file type: %s", de)
+			continue
 		}
+		ret = append(ret, obj)
 	}
 
 	return
@@ -192,6 +200,14 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 				"Result":         didlLite(string(result)),
 				"UpdateID":       cds.updateIDString(),
 			}, nil
+		case "BrowseMetadata":
+			result, err := xml.Marshal(obj)
+			if err != nil {
+				return nil, err
+			}
+			return map[string]string{
+				"Result": didlLite(string(result)),
+			}, nil
 		default:
 			return nil, upnp.Errorf(upnp.ArgumentValueInvalidErrorCode, "unhandled browse flag: %v", browse.BrowseFlag)
 		}
@@ -199,6 +215,19 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 		return map[string]string{
 			"SearchCaps": "",
 		}, nil
+	// Samsung Extensions
+	case "X_GetFeatureList":
+		return map[string]string{
+			"FeatureList": `<Features xmlns="urn:schemas-upnp-org:av:avs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:schemas-upnp-org:av:avs http://www.upnp.org/schemas/av/avs.xsd">
+	<Feature name="samsung.com_BASICVIEW" version="1">
+		<container id="/" type="object.item.imageItem"/>
+		<container id="/" type="object.item.audioItem"/>
+		<container id="/" type="object.item.videoItem"/>
+	</Feature>
+</Features>`}, nil
+	case "X_SetBookmark":
+		// just ignore
+		return map[string]string{}, nil
 	default:
 		return nil, upnp.InvalidActionError
 	}

cmd/serve/dlna/cms.go

@@ -0,0 +1,26 @@
+package dlna
+
+import (
+	"net/http"
+
+	"github.com/anacrolix/dms/upnp"
+)
+
+const defaultProtocolInfo = "http-get:*:video/mpeg:*,http-get:*:video/mp4:*,http-get:*:video/vnd.dlna.mpeg-tts:*,http-get:*:video/avi:*,http-get:*:video/x-matroska:*,http-get:*:video/x-ms-wmv:*,http-get:*:video/wtv:*,http-get:*:audio/mpeg:*,http-get:*:audio/mp3:*,http-get:*:audio/mp4:*,http-get:*:audio/x-ms-wma*,http-get:*:audio/wav:*,http-get:*:audio/L16:*,http-get:*image/jpeg:*,http-get:*image/png:*,http-get:*image/gif:*,http-get:*image/tiff:*"
+
+type connectionManagerService struct {
+	*server
+	upnp.Eventing
+}
+
+func (cms *connectionManagerService) Handle(action string, argsXML []byte, r *http.Request) (map[string]string, error) {
+	switch action {
+	case "GetProtocolInfo":
+		return map[string]string{
+			"Source": defaultProtocolInfo,
+			"Sink":   "",
+		}, nil
+	default:
+		return nil, upnp.InvalidActionError
+	}
+}

cmd/serve/dlna/data/assets_generate.go

@@ -0,0 +1,24 @@
+//go:generate go run assets_generate.go
+// The "go:generate" directive compiles static assets by running assets_generate.go
+// +build ignore
+
+package main
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/shurcooL/vfsgen"
+)
+
+func main() {
+	var AssetDir http.FileSystem = http.Dir("./static")
+	err := vfsgen.Generate(AssetDir, vfsgen.Options{
+		PackageName:  "data",
+		BuildTags:    "!dev",
+		VariableName: "Assets",
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

cmd/serve/dlna/data/data.go

@@ -0,0 +1,4 @@
+//go:generate go run assets_generate.go
+// The "go:generate" directive compiles static assets by running assets_generate.go
+
+package data

cmd/serve/dlna/data/static/ConnectionManager.xml

@@ -0,0 +1,182 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>GetProtocolInfo</name>
+			<argumentList>
+				<argument>
+					<name>Source</name>
+					<direction>out</direction>
+					<relatedStateVariable>SourceProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Sink</name>
+					<direction>out</direction>
+					<relatedStateVariable>SinkProtocolInfo</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>PrepareForConnection</name>
+			<argumentList>
+				<argument>
+					<name>RemoteProtocolInfo</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionManager</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Direction</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>AVTransportID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RcsID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ConnectionComplete</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetCurrentConnectionIDs</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionIDs</name>
+					<direction>out</direction>
+					<relatedStateVariable>CurrentConnectionIDs</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetCurrentConnectionInfo</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RcsID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>AVTransportID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ProtocolInfo</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionManager</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Direction</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Status</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionStatus</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="yes">
+			<name>SourceProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>SinkProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>CurrentConnectionIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionStatus</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>OK</allowedValue>
+				<allowedValue>ContentFormatMismatch</allowedValue>
+				<allowedValue>InsufficientBandwidth</allowedValue>
+				<allowedValue>UnreliableChannel</allowedValue>
+				<allowedValue>Unknown</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionManager</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Direction</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>Input</allowedValue>
+				<allowedValue>Output</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_AVTransportID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RcsID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/data/static/ContentDirectory.xml

@@ -0,0 +1,504 @@
+<?xml version="1.0"?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>GetSearchCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SearchCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SearchCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSortCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SortCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SortCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSortExtensionCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SortExtensionCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetFeatureList</name>
+			<argumentList>
+				<argument>
+					<name>FeatureList</name>
+					<direction>out</direction>
+					<relatedStateVariable>FeatureList</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSystemUpdateID</name>
+			<argumentList>
+				<argument>
+					<name>Id</name>
+					<direction>out</direction>
+					<relatedStateVariable>SystemUpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>Browse</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>BrowseFlag</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Filter</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>StartingIndex</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RequestedCount</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SortCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NumberReturned</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TotalMatches</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>UpdateID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>Search</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SearchCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Filter</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>StartingIndex</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RequestedCount</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SortCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NumberReturned</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TotalMatches</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>UpdateID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>CreateObject</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Elements</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>DestroyObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>UpdateObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>CurrentTagValue</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewTagValue</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>MoveObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewParentID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewObjectID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ImportResource</name>
+			<argumentList>
+				<argument>
+					<name>SourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>DestinationURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ExportResource</name>
+			<argumentList>
+				<argument>
+					<name>SourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>DestinationURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>StopTransferResource</name>
+			<argumentList>
+				<argument>
+					<name>TransferID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>DeleteResource</name>
+			<argumentList>
+				<argument>
+					<name>ResourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetTransferProgress</name>
+			<argumentList>
+				<argument>
+					<name>TransferID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferStatus</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferLength</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferTotal</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>CreateReference</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>X_GetFeatureList</name>
+			<argumentList>
+				<argument>
+					<name>FeatureList</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Featurelist</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>X_SetBookmark</name>
+			<argumentList>
+				<argument>
+					<name>CategoryType</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_CategoryType</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_RID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PosSecond</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_PosSec</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="no">
+			<name>SearchCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>SortCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>SortExtensionCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>SystemUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ContainerUpdateIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>TransferIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>FeatureList</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ObjectID</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Result</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_SearchCriteria</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_BrowseFlag</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>BrowseMetadata</allowedValue>
+				<allowedValue>BrowseDirectChildren</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Filter</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_SortCriteria</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Index</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Count</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_UpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferStatus</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>COMPLETED</allowedValue>
+				<allowedValue>ERROR</allowedValue>
+				<allowedValue>IN_PROGRESS</allowedValue>
+				<allowedValue>STOPPED</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferLength</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferTotal</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TagValueList</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_URI</name>
+			<dataType>uri</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_CategoryType</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RID</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_PosSec</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Featurelist</name>
+			<dataType>string</dataType>
+			<defaultValue />
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/data/static/X_MS_MediaReceiverRegistrar.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" ?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>IsAuthorized</name>
+			<argumentList>
+				<argument>
+					<name>DeviceID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_DeviceID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>RegisterDevice</name>
+			<argumentList>
+				<argument>
+					<name>RegistrationReqMsg</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_RegistrationReqMsg</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RegistrationRespMsg</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RegistrationRespMsg</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>IsValidated</name>
+			<argumentList>
+				<argument>
+					<name>DeviceID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_DeviceID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_DeviceID</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Result</name>
+			<dataType>int</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RegistrationReqMsg</name>
+			<dataType>bin.base64</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RegistrationRespMsg</name>
+			<dataType>bin.base64</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>AuthorizationGrantedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>AuthorizationDeniedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ValidationSucceededUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ValidationRevokedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/dlna.go

@@ -4,20 +4,21 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
+	dms_dlna "github.com/anacrolix/dms/dlna"
 	"github.com/anacrolix/dms/soap"
 	"github.com/anacrolix/dms/ssdp"
 	"github.com/anacrolix/dms/upnp"
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/dlna/data"
 	"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/vfs"
@@ -51,7 +52,7 @@ players might show files that they are not able to play back correctly.
 		cmd.Run(false, false, command, func() error {
 			s := newServer(f, &dlnaflags.Opt)
 			if err := s.Serve(); err != nil {
-				log.Fatal(err)
+				return err
 			}
 			s.Wait()
 			return nil
@@ -60,45 +61,12 @@ players might show files that they are not able to play back correctly.
 }
 
 const (
-	serverField         = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
-	rootDeviceType      = "urn:schemas-upnp-org:device:MediaServer:1"
-	rootDeviceModelName = "rclone"
-	resPath             = "/res"
-	rootDescPath        = "/rootDesc.xml"
-	serviceControlURL   = "/ctl"
+	serverField       = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
+	rootDescPath      = "/rootDesc.xml"
+	resPath           = "/res"
+	serviceControlURL = "/ctl"
 )
 
-// Groups the service definition with its XML description.
-type service struct {
-	upnp.Service
-	SCPD string
-}
-
-// Exposed UPnP AV services.
-var services = []*service{
-	{
-		Service: upnp.Service{
-			ServiceType: "urn:schemas-upnp-org:service:ContentDirectory:1",
-			ServiceId:   "urn:upnp-org:serviceId:ContentDirectory",
-			ControlURL:  serviceControlURL,
-		},
-		SCPD: contentDirectoryServiceDescription,
-	},
-}
-
-func devices() []string {
-	return []string{
-		"urn:schemas-upnp-org:device:MediaServer:1",
-	}
-}
-
-func serviceTypes() (ret []string) {
-	for _, s := range services {
-		ret = append(ret, s.ServiceType)
-	}
-	return
-}
-
 type server struct {
 	// The service SOAP handler keyed by service URN.
 	services map[string]UPnPService
@@ -107,10 +75,9 @@ type server struct {
 
 	HTTPConn       net.Listener
 	httpListenAddr string
-	httpServeMux   *http.ServeMux
+	handler        http.Handler
 
-	rootDeviceUUID string
-	rootDescXML    []byte
+	RootDeviceUUID string
 
 	FriendlyName string
 
@@ -125,16 +92,16 @@ type server struct {
 }
 
 func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
-	hostName, err := os.Hostname()
-	if err != nil {
-		hostName = ""
-	} else {
-		hostName = " (" + hostName + ")"
+	friendlyName := opt.FriendlyName
+	if friendlyName == "" {
+		friendlyName = makeDefaultFriendlyName()
 	}
 
 	s := &server{
 		AnnounceInterval: 10 * time.Second,
-		FriendlyName:     "rclone" + hostName,
+		FriendlyName:     friendlyName,
+		RootDeviceUUID:   makeDeviceUUID(friendlyName),
+		Interfaces:       listInterfaces(),
 
 		httpListenAddr: opt.ListenAddr,
 
@@ -142,35 +109,29 @@ func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
 		vfs: vfs.New(f, &vfsflags.Opt),
 	}
 
-	s.initServicesMap()
-	s.listInterfaces()
-
-	s.httpServeMux = http.NewServeMux()
-	s.rootDeviceUUID = makeDeviceUUID(s.FriendlyName)
-	s.rootDescXML, err = xml.MarshalIndent(
-		upnp.DeviceDesc{
-			SpecVersion: upnp.SpecVersion{Major: 1, Minor: 0},
-			Device: upnp.Device{
-				DeviceType:   rootDeviceType,
-				FriendlyName: s.FriendlyName,
-				Manufacturer: "rclone (rclone.org)",
-				ModelName:    rootDeviceModelName,
-				UDN:          s.rootDeviceUUID,
-				ServiceList: func() (ss []upnp.Service) {
-					for _, s := range services {
-						ss = append(ss, s.Service)
-					}
-					return
-				}(),
-			},
+	s.services = map[string]UPnPService{
+		"ContentDirectory": &contentDirectoryService{
+			server: s,
+		},
+		"ConnectionManager": &connectionManagerService{
+			server: s,
 		},
-		" ", "  ")
-	if err != nil {
-		// Contents are hardcoded, so this will never happen in production.
-		log.Panicf("Marshal root descriptor XML: %v", err)
 	}
-	s.rootDescXML = append([]byte(`<?xml version="1.0"?>`), s.rootDescXML...)
-	s.initMux(s.httpServeMux)
+
+	// Setup the various http routes.
+	r := http.NewServeMux()
+	r.HandleFunc(resPath, s.resourceHandler)
+	if opt.LogTrace {
+		r.Handle(rootDescPath, traceLogging(http.HandlerFunc(s.rootDescHandler)))
+		r.Handle(serviceControlURL, traceLogging(http.HandlerFunc(s.serviceControlHandler)))
+	} else {
+		r.HandleFunc(rootDescPath, s.rootDescHandler)
+		r.HandleFunc(serviceControlURL, s.serviceControlHandler)
+	}
+	r.Handle("/static/", http.StripPrefix("/static/",
+		withHeader("Cache-Control", "public, max-age=86400",
+			http.FileServer(data.Assets))))
+	s.handler = logging(withHeader("Server", serverField, r))
 
 	return s
 }
@@ -182,117 +143,137 @@ type UPnPService interface {
 	Unsubscribe(sid string) error
 }
 
-// initServicesMap is called during initialization of the server to prepare some internal datastructures.
-func (s *server) initServicesMap() {
-	urn, err := upnp.ParseServiceType(services[0].ServiceType)
-	if err != nil {
-		// The service type is hardcoded, so this error should never happen.
-		log.Panicf("ParseServiceType: %v", err)
-	}
-	s.services = map[string]UPnPService{
-		urn.Type: &contentDirectoryService{
-			server: s,
-		},
-	}
-	return
+// Formats the server as a string (used for logging.)
+func (s *server) String() string {
+	return fmt.Sprintf("DLNA server on %v", s.httpListenAddr)
 }
 
-// listInterfaces is called during initialization of the server to list the network interfaces
-// on the machine.
-func (s *server) listInterfaces() {
-	ifs, err := net.Interfaces()
+// Returns rclone version number as the model number.
+func (s *server) ModelNumber() string {
+	return fs.Version
+}
+
+// Template used to generate the root device XML descriptor.
+//
+// Due to the use of namespaces and various subtleties with device compatibility,
+// it turns out to be easier to use a template than to marshal XML.
+//
+// For rendering, it is passed the server object for context.
+var rootDescTmpl = template.Must(template.New("rootDesc").Parse(`<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"
+      xmlns:dlna="urn:schemas-dlna-org:device-1-0"
+      xmlns:sec="http://www.sec.co.kr/dlna">
+  <specVersion>
+    <major>1</major>
+    <minor>0</minor>
+  </specVersion>
+  <device>
+    <deviceType>urn:schemas-upnp-org:device:MediaServer:1</deviceType>
+    <friendlyName>{{.FriendlyName}}</friendlyName>
+    <manufacturer>rclone (rclone.org)</manufacturer>
+    <manufacturerURL>https://rclone.org/</manufacturerURL>
+    <modelDescription>rclone</modelDescription>
+    <modelName>rclone</modelName>
+    <modelNumber>{{.ModelNumber}}</modelNumber>
+    <modelURL>https://rclone.org/</modelURL>
+    <serialNumber>00000000</serialNumber>
+    <UDN>{{.RootDeviceUUID}}</UDN>
+    <dlna:X_DLNACAP/>
+    <dlna:X_DLNADOC>DMS-1.50</dlna:X_DLNADOC>
+    <dlna:X_DLNADOC>M-DMS-1.50</dlna:X_DLNADOC>
+    <sec:ProductCap>smi,DCM10,getMediaInfo.sec,getCaptionInfo.sec</sec:ProductCap>
+    <sec:X_ProductCap>smi,DCM10,getMediaInfo.sec,getCaptionInfo.sec</sec:X_ProductCap>
+    <iconList>
+      <icon>
+        <mimetype>image/png</mimetype>
+        <width>48</width>
+        <height>48</height>
+        <depth>8</depth>
+        <url>/static/rclone-48x48.png</url>
+      </icon>
+      <icon>
+        <mimetype>image/png</mimetype>
+        <width>120</width>
+        <height>120</height>
+        <depth>8</depth>
+        <url>/static/rclone-120x120.png</url>
+      </icon>
+    </iconList>
+    <serviceList>
+      <service>
+        <serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>
+        <serviceId>urn:upnp-org:serviceId:ContentDirectory</serviceId>
+        <SCPDURL>/static/ContentDirectory.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+      <service>
+        <serviceType>urn:schemas-upnp-org:service:ConnectionManager:1</serviceType>
+        <serviceId>urn:upnp-org:serviceId:ConnectionManager</serviceId>
+        <SCPDURL>/static/ConnectionManager.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+      <service>
+        <serviceType>urn:microsoft.com:service:X_MS_MediaReceiverRegistrar:1</serviceType>
+        <serviceId>urn:microsoft.com:serviceId:X_MS_MediaReceiverRegistrar</serviceId>
+        <SCPDURL>/static/X_MS_MediaReceiverRegistrar.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+    </serviceList>
+    <presentationURL>/</presentationURL>
+  </device>
+</root>`))
+
+// Renders the root device descriptor.
+func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {
+	buffer := new(bytes.Buffer)
+	err := rootDescTmpl.Execute(buffer, s)
 	if err != nil {
-		fs.Errorf(s.f, "list network interfaces: %v", err)
+		serveError(s, w, "Failed to create root descriptor XML", err)
 		return
 	}
 
-	var tmp []net.Interface
-	for _, intf := range ifs {
-		if intf.Flags&net.FlagUp == 0 || intf.MTU <= 0 {
-			continue
-		}
-		s.Interfaces = append(s.Interfaces, intf)
-		tmp = append(tmp, intf)
+	w.Header().Set("content-type", `text/xml; charset="utf-8"`)
+	w.Header().Set("cache-control", "private, max-age=60")
+	w.Header().Set("content-length", strconv.FormatInt(int64(buffer.Len()), 10))
+	_, err = buffer.WriteTo(w)
+	if err != nil {
+		// Network error
+		fs.Debugf(s, "Error writing rootDesc: %v", err)
 	}
 }
 
-func (s *server) initMux(mux *http.ServeMux) {
-	mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) {
-		remotePath := r.URL.Query().Get("path")
-		node, err := s.vfs.Stat(remotePath)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
-
-		file := node.(*vfs.File)
-		in, err := file.Open(os.O_RDONLY)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-		defer fs.CheckClose(in, &err)
-
-		http.ServeContent(w, r, remotePath, node.ModTime(), in)
-		return
-	})
-
-	mux.HandleFunc(rootDescPath, func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-		w.Header().Set("content-length", fmt.Sprint(len(s.rootDescXML)))
-		w.Header().Set("server", serverField)
-		_, err := w.Write(s.rootDescXML)
-		if err != nil {
-			fs.Errorf(s, "Failed to serve root descriptor XML: %v", err)
-		}
-	})
-
-	// Install handlers to serve SCPD for each UPnP service.
-	for _, s := range services {
-		p := path.Join("/scpd", s.ServiceId)
-		s.SCPDURL = p
-
-		mux.HandleFunc(s.SCPDURL, func(serviceDesc string) http.HandlerFunc {
-			return func(w http.ResponseWriter, r *http.Request) {
-				w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-				http.ServeContent(w, r, ".xml", time.Time{}, bytes.NewReader([]byte(serviceDesc)))
-			}
-		}(s.SCPD))
-	}
-
-	mux.HandleFunc(serviceControlURL, s.serviceControlHandler)
-}
-
 // Handle a service control HTTP request.
 func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {
 	soapActionString := r.Header.Get("SOAPACTION")
 	soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		serveError(s, w, "Could not parse SOAPACTION header", err)
 		return
 	}
 	var env soap.Envelope
 	if err := xml.NewDecoder(r.Body).Decode(&env); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		serveError(s, w, "Could not parse SOAP request body", err)
 		return
 	}
 
 	w.Header().Set("Content-Type", `text/xml; charset="utf-8"`)
 	w.Header().Set("Ext", "")
-	w.Header().Set("server", serverField)
 	soapRespXML, code := func() ([]byte, int) {
 		respArgs, err := s.soapActionResponse(soapAction, env.Body.Action, r)
 		if err != nil {
+			fs.Errorf(s, "Error invoking %v: %v", soapAction, err)
 			upnpErr := upnp.ConvertError(err)
-			return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), 500
+			return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), http.StatusInternalServerError
 		}
-		return marshalSOAPResponse(soapAction, respArgs), 200
+		return marshalSOAPResponse(soapAction, respArgs), http.StatusOK
 	}()
 	bodyStr := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" standalone="yes"?><s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"><s:Body>%s</s:Body></s:Envelope>`, soapRespXML)
 	w.WriteHeader(code)
 	if _, err := w.Write([]byte(bodyStr)); err != nil {
-		log.Print(err)
+		fs.Infof(s, "Error writing response: %v", err)
 	}
 }
 
@@ -306,6 +287,36 @@ func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte,
 	return service.Handle(sa.Action, actionRequestXML, r)
 }
 
+// Serves actual resources (media files).
+func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {
+	remotePath := r.URL.Query().Get("path")
+	node, err := s.vfs.Stat(remotePath)
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
+
+	// add some DLNA specific headers
+	if r.Header.Get("getContentFeatures.dlna.org") != "" {
+		w.Header().Set("contentFeatures.dlna.org", dms_dlna.ContentFeatures{
+			SupportRange: true,
+		}.String())
+	}
+	w.Header().Set("transferMode.dlna.org", "Streaming")
+
+	file := node.(*vfs.File)
+	in, err := file.Open(os.O_RDONLY)
+	if err != nil {
+		serveError(node, w, "Could not open resource", err)
+		return
+	}
+	defer fs.CheckClose(in, &err)
+
+	http.ServeContent(w, r, remotePath, node.ModTime(), in)
+}
+
 // Serve runs the server - returns the error only if
 // the listener was not started; does not block, so
 // use s.Wait() to block on the listener indefinitely.
@@ -381,13 +392,19 @@ func (s *server) ssdpInterface(intf net.Interface) {
 		return url.String()
 	}
 
+	// Note that the devices and services advertised here via SSDP should be
+	// in agreement with the rootDesc XML descriptor that is defined above.
 	ssdpServer := ssdp.Server{
-		Interface:      intf,
-		Devices:        devices(),
-		Services:       serviceTypes(),
+		Interface: intf,
+		Devices: []string{
+			"urn:schemas-upnp-org:device:MediaServer:1"},
+		Services: []string{
+			"urn:schemas-upnp-org:service:ContentDirectory:1",
+			"urn:schemas-upnp-org:service:ConnectionManager:1",
+			"urn:microsoft.com:service:X_MS_MediaReceiverRegistrar:1"},
 		Location:       advertiseLocationFn,
 		Server:         serverField,
-		UUID:           s.rootDeviceUUID,
+		UUID:           s.RootDeviceUUID,
 		NotifyInterval: s.AnnounceInterval,
 	}
 
@@ -405,16 +422,16 @@ func (s *server) ssdpInterface(intf net.Interface) {
 			// good.
 			return
 		}
-		log.Printf("Error creating ssdp server on %s: %s", intf.Name, err)
+		fs.Errorf(s, "Error creating ssdp server on %s: %s", intf.Name, err)
 		return
 	}
 	defer ssdpServer.Close()
-	log.Println("Started SSDP on", intf.Name)
+	fs.Infof(s, "Started SSDP on %v", intf.Name)
 	stopped := make(chan struct{})
 	go func() {
 		defer close(stopped)
 		if err := ssdpServer.Serve(); err != nil {
-			log.Printf("%q: %q\n", intf.Name, err)
+			fs.Errorf(s, "%q: %q\n", intf.Name, err)
 		}
 	}()
 	select {
@@ -426,9 +443,7 @@ func (s *server) ssdpInterface(intf net.Interface) {
 
 func (s *server) serveHTTP() error {
 	srv := &http.Server{
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			s.httpServeMux.ServeHTTP(w, r)
-		}),
+		Handler: s.handler,
 	}
 	err := srv.Serve(s.HTTPConn)
 	select {

cmd/serve/dlna/dlna_test.go

@@ -1,5 +1,3 @@
-// +build go1.8
-
 package dlna
 
 import (
@@ -22,11 +20,11 @@ import (
 
 var (
 	dlnaServer *server
+	testURL    string
 )
 
 const (
-	testBindAddress = "localhost:51777"
-	testURL         = "http://" + testBindAddress + "/"
+	testBindAddress = "localhost:0"
 )
 
 func startServer(t *testing.T, f fs.Fs) {
@@ -34,6 +32,7 @@ func startServer(t *testing.T, f fs.Fs) {
 	opt.ListenAddr = testBindAddress
 	dlnaServer = newServer(f, &opt)
 	assert.NoError(t, dlnaServer.Serve())
+	testURL = "http://" + dlnaServer.HTTPConn.Addr().String() + "/"
 }
 
 func TestInit(t *testing.T) {
@@ -59,6 +58,11 @@ func TestRootSCPD(t *testing.T) {
 	// Make sure that the SCPD contains a CDS service.
 	require.Contains(t, string(body),
 		"<serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>")
+	// Make sure that the SCPD contains a CM service.
+	require.Contains(t, string(body),
+		"<serviceType>urn:schemas-upnp-org:service:ConnectionManager:1</serviceType>")
+	// Ensure that the SCPD url is configured.
+	require.Regexp(t, "<SCPDURL>/.*</SCPDURL>", string(body))
 }
 
 // Make sure that it serves content from the remote.